{
 "cells": [
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Quick Start",
   "id": "d02cbadbc5704c8b"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-09T15:03:17.061740Z",
     "start_time": "2024-10-09T15:00:18.736065Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets\n",
    "from torchvision.transforms import ToTensor\n",
    "\n",
    "# Download training data from open datasets.\n",
    "training_data = datasets.FashionMNIST(\n",
    "    root=\"data\",\n",
    "    train=True,\n",
    "    download=True,\n",
    "    transform=ToTensor(),\n",
    ")\n",
    "\n",
    "# Download test data from open datasets.\n",
    "test_data = datasets.FashionMNIST(\n",
    "    root=\"data\",\n",
    "    train=False,\n",
    "    download=True,\n",
    "    transform=ToTensor(),\n",
    ")"
   ],
   "id": "a9fa0ebbd8db1f69",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz\n",
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz to data/FashionMNIST/raw/train-images-idx3-ubyte.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100.0%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/FashionMNIST/raw/train-images-idx3-ubyte.gz to data/FashionMNIST/raw\n",
      "\n",
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz\n",
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz to data/FashionMNIST/raw/train-labels-idx1-ubyte.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100.0%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/FashionMNIST/raw/train-labels-idx1-ubyte.gz to data/FashionMNIST/raw\n",
      "\n",
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz\n",
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz to data/FashionMNIST/raw/t10k-images-idx3-ubyte.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100.0%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/FashionMNIST/raw/t10k-images-idx3-ubyte.gz to data/FashionMNIST/raw\n",
      "\n",
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz\n",
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz to data/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100.0%"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz to data/FashionMNIST/raw\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-09T15:04:39.983494Z",
     "start_time": "2024-10-09T15:04:39.962232Z"
    }
   },
   "cell_type": "code",
   "source": [
    "batch_size = 64\n",
    "\n",
    "# Create data loaders.\n",
    "train_dataloader = DataLoader(training_data, batch_size=batch_size)\n",
    "test_dataloader = DataLoader(test_data, batch_size=batch_size)\n",
    "\n",
    "for X, y in test_dataloader:\n",
    "    print(f\"Shape of X [N, C, H, W]: {X.shape}\")\n",
    "    print(f\"Shape of y: {y.shape} {y.dtype}\")\n",
    "    break"
   ],
   "id": "a6d05b33d4f2c0ac",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of X [N, C, H, W]: torch.Size([64, 1, 28, 28])\n",
      "Shape of y: torch.Size([64]) torch.int64\n"
     ]
    }
   ],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-09T15:04:44.477698Z",
     "start_time": "2024-10-09T15:04:44.403109Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# Get cpu, gpu or mps device for training.\n",
    "device = (\n",
    "    \"cuda\"\n",
    "    if torch.cuda.is_available()\n",
    "    else \"mps\"\n",
    "    if torch.backends.mps.is_available()\n",
    "    else \"cpu\"\n",
    ")\n",
    "print(f\"Using {device} device\")\n",
    "\n",
    "# Define model\n",
    "class NeuralNetwork(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.flatten = nn.Flatten()\n",
    "        self.linear_relu_stack = nn.Sequential(\n",
    "            nn.Linear(28*28, 512),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(512, 512),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(512, 10)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.flatten(x)\n",
    "        logits = self.linear_relu_stack(x)\n",
    "        return logits\n",
    "\n",
    "model = NeuralNetwork().to(device)\n",
    "print(model)"
   ],
   "id": "1e2ccb023ca97e9c",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using mps device\n",
      "NeuralNetwork(\n",
      "  (flatten): Flatten(start_dim=1, end_dim=-1)\n",
      "  (linear_relu_stack): Sequential(\n",
      "    (0): Linear(in_features=784, out_features=512, bias=True)\n",
      "    (1): ReLU()\n",
      "    (2): Linear(in_features=512, out_features=512, bias=True)\n",
      "    (3): ReLU()\n",
      "    (4): Linear(in_features=512, out_features=10, bias=True)\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-09T15:11:43.794018Z",
     "start_time": "2024-10-09T15:11:27.359935Z"
    }
   },
   "cell_type": "code",
   "source": [
    "loss_fn = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=1e-3)\n",
    "\n",
    "def train(dataloader, model, loss_fn, optimizer):\n",
    "    size = len(dataloader.dataset)\n",
    "    model.train()\n",
    "    for batch, (X, y) in enumerate(dataloader):\n",
    "        X, y = X.to(device), y.to(device)\n",
    "\n",
    "        # Compute prediction error\n",
    "        pred = model(X)\n",
    "        loss = loss_fn(pred, y)\n",
    "\n",
    "        # Backpropagation\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        if batch % 100 == 0:\n",
    "            loss, current = loss.item(), (batch + 1) * len(X)\n",
    "            print(f\"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]\")\n",
    "            \n",
    "def test(dataloader, model, loss_fn):\n",
    "    size = len(dataloader.dataset)\n",
    "    num_batches = len(dataloader)\n",
    "    model.eval()\n",
    "    test_loss, correct = 0, 0\n",
    "    with torch.no_grad():\n",
    "        for X, y in dataloader:\n",
    "            X, y = X.to(device), y.to(device)\n",
    "            pred = model(X)\n",
    "            test_loss += loss_fn(pred, y).item()\n",
    "            correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
    "    test_loss /= num_batches\n",
    "    correct /= size\n",
    "    print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
    "    \n",
    "epochs = 5\n",
    "for t in range(epochs):\n",
    "    print(f\"Epoch {t+1}\\n-------------------------------\")\n",
    "    train(train_dataloader, model, loss_fn, optimizer)\n",
    "    test(test_dataloader, model, loss_fn)\n",
    "print(\"Done!\")"
   ],
   "id": "4d782adbf9aa2277",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 2.315795  [   64/60000]\n",
      "loss: 2.300760  [ 6464/60000]\n",
      "loss: 2.277517  [12864/60000]\n",
      "loss: 2.264001  [19264/60000]\n",
      "loss: 2.247344  [25664/60000]\n",
      "loss: 2.216080  [32064/60000]\n",
      "loss: 2.233718  [38464/60000]\n",
      "loss: 2.200001  [44864/60000]\n",
      "loss: 2.200125  [51264/60000]\n",
      "loss: 2.148623  [57664/60000]\n",
      "Test Error: \n",
      " Accuracy: 35.9%, Avg loss: 2.150771 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 2.168602  [   64/60000]\n",
      "loss: 2.157149  [ 6464/60000]\n",
      "loss: 2.095682  [12864/60000]\n",
      "loss: 2.107099  [19264/60000]\n",
      "loss: 2.053087  [25664/60000]\n",
      "loss: 1.993099  [32064/60000]\n",
      "loss: 2.025957  [38464/60000]\n",
      "loss: 1.947560  [44864/60000]\n",
      "loss: 1.953955  [51264/60000]\n",
      "loss: 1.864054  [57664/60000]\n",
      "Test Error: \n",
      " Accuracy: 57.3%, Avg loss: 1.871236 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 1.905457  [   64/60000]\n",
      "loss: 1.875705  [ 6464/60000]\n",
      "loss: 1.757384  [12864/60000]\n",
      "loss: 1.799198  [19264/60000]\n",
      "loss: 1.694123  [25664/60000]\n",
      "loss: 1.639552  [32064/60000]\n",
      "loss: 1.668348  [38464/60000]\n",
      "loss: 1.572725  [44864/60000]\n",
      "loss: 1.592200  [51264/60000]\n",
      "loss: 1.488058  [57664/60000]\n",
      "Test Error: \n",
      " Accuracy: 62.3%, Avg loss: 1.511620 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 1.569320  [   64/60000]\n",
      "loss: 1.542129  [ 6464/60000]\n",
      "loss: 1.396262  [12864/60000]\n",
      "loss: 1.470897  [19264/60000]\n",
      "loss: 1.365247  [25664/60000]\n",
      "loss: 1.347719  [32064/60000]\n",
      "loss: 1.373919  [38464/60000]\n",
      "loss: 1.299059  [44864/60000]\n",
      "loss: 1.329065  [51264/60000]\n",
      "loss: 1.233871  [57664/60000]\n",
      "Test Error: \n",
      " Accuracy: 63.5%, Avg loss: 1.260619 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 1.327890  [   64/60000]\n",
      "loss: 1.316783  [ 6464/60000]\n",
      "loss: 1.154818  [12864/60000]\n",
      "loss: 1.259650  [19264/60000]\n",
      "loss: 1.147469  [25664/60000]\n",
      "loss: 1.156365  [32064/60000]\n",
      "loss: 1.191788  [38464/60000]\n",
      "loss: 1.126612  [44864/60000]\n",
      "loss: 1.161734  [51264/60000]\n",
      "loss: 1.077984  [57664/60000]\n",
      "Test Error: \n",
      " Accuracy: 64.6%, Avg loss: 1.099939 \n",
      "\n",
      "Done!\n"
     ]
    }
   ],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-09T15:12:34.571417Z",
     "start_time": "2024-10-09T15:12:34.569169Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import os\n",
    "\n",
    "os.mkdir(\"model\")"
   ],
   "id": "a404134783294e30",
   "outputs": [],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-09T15:13:08.213455Z",
     "start_time": "2024-10-09T15:13:08.198505Z"
    }
   },
   "cell_type": "code",
   "source": [
    "torch.save(model.state_dict(), \"model/model.pth\")\n",
    "print(\"Saved PyTorch Model State to model.pth\")"
   ],
   "id": "8adf8c0f968201b7",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved PyTorch Model State to model.pth\n"
     ]
    }
   ],
   "execution_count": 6
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-09T15:14:09.675179Z",
     "start_time": "2024-10-09T15:14:09.660797Z"
    }
   },
   "cell_type": "code",
   "source": [
    "model = NeuralNetwork().to(device)\n",
    "model.load_state_dict(torch.load(\"model/model.pth\", weights_only=True))"
   ],
   "id": "31dd352bc199cde",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 7
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-09T15:15:17.533916Z",
     "start_time": "2024-10-09T15:15:17.268399Z"
    }
   },
   "cell_type": "code",
   "source": [
    "classes = [\n",
    "    \"T-shirt/top\",\n",
    "    \"Trouser\",\n",
    "    \"Pullover\",\n",
    "    \"Dress\",\n",
    "    \"Coat\",\n",
    "    \"Sandal\",\n",
    "    \"Shirt\",\n",
    "    \"Sneaker\",\n",
    "    \"Bag\",\n",
    "    \"Ankle boot\",\n",
    "]\n",
    "\n",
    "model.eval()\n",
    "x, y = test_data[0][0], test_data[0][1]\n",
    "with torch.no_grad():\n",
    "    x = x.to(device)\n",
    "    pred = model(x)\n",
    "    predicted, actual = classes[pred[0].argmax(0)], classes[y]\n",
    "    print(f'Predicted: \"{predicted}\", Actual: \"{actual}\"')"
   ],
   "id": "5f8e989dab557196",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted: \"Ankle boot\", Actual: \"Ankle boot\"\n"
     ]
    }
   ],
   "execution_count": 8
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Tensors",
   "id": "ae3fa6eb4e5af55d"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-10T14:15:32.134716Z",
     "start_time": "2024-10-10T14:15:31.318147Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import torch\n",
    "import numpy as np"
   ],
   "id": "40b0d5a17e957329",
   "outputs": [],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-10T14:16:19.410717Z",
     "start_time": "2024-10-10T14:16:19.408376Z"
    }
   },
   "cell_type": "code",
   "source": [
    "data = [[1, 2],[3, 4]]\n",
    "x_data = torch.tensor(data)"
   ],
   "id": "a04d0874f49ab6f3",
   "outputs": [],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-10T14:17:00.033982Z",
     "start_time": "2024-10-10T14:17:00.031969Z"
    }
   },
   "cell_type": "code",
   "source": [
    "np_array = np.array(data)\n",
    "x_np = torch.from_numpy(np_array)"
   ],
   "id": "c6f89569522d3490",
   "outputs": [],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-10T14:17:50.421899Z",
     "start_time": "2024-10-10T14:17:50.417380Z"
    }
   },
   "cell_type": "code",
   "source": [
    "x_ones = torch.ones_like(x_data) # retains the properties of x_data\n",
    "print(f\"Ones Tensor: \\n {x_ones} \\n\")\n",
    "\n",
    "x_rand = torch.rand_like(x_data, dtype=torch.float) # overrides the datatype of x_data\n",
    "print(f\"Random Tensor: \\n {x_rand} \\n\")"
   ],
   "id": "38bc57b5a8c280c",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ones Tensor: \n",
      " tensor([[1, 1],\n",
      "        [1, 1]]) \n",
      "\n",
      "Random Tensor: \n",
      " tensor([[0.4594, 0.8803],\n",
      "        [0.6965, 0.8204]]) \n",
      "\n"
     ]
    }
   ],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-10T14:19:02.853497Z",
     "start_time": "2024-10-10T14:19:02.849495Z"
    }
   },
   "cell_type": "code",
   "source": [
    "shape = (2,3,)\n",
    "rand_tensor = torch.rand(shape)\n",
    "ones_tensor = torch.ones(shape)\n",
    "zeros_tensor = torch.zeros(shape)\n",
    "\n",
    "print(f\"Random Tensor: \\n {rand_tensor} \\n\")\n",
    "print(f\"Ones Tensor: \\n {ones_tensor} \\n\")\n",
    "print(f\"Zeros Tensor: \\n {zeros_tensor}\")"
   ],
   "id": "6f06a142d86af8cd",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Tensor: \n",
      " tensor([[0.9156, 0.7763, 0.4820],\n",
      "        [0.9833, 0.5919, 0.7940]]) \n",
      "\n",
      "Ones Tensor: \n",
      " tensor([[1., 1., 1.],\n",
      "        [1., 1., 1.]]) \n",
      "\n",
      "Zeros Tensor: \n",
      " tensor([[0., 0., 0.],\n",
      "        [0., 0., 0.]])\n"
     ]
    }
   ],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-10T14:19:31.833280Z",
     "start_time": "2024-10-10T14:19:31.830040Z"
    }
   },
   "cell_type": "code",
   "source": [
    "tensor = torch.rand(3,4)\n",
    "\n",
    "print(f\"Shape of tensor: {tensor.shape}\")\n",
    "print(f\"Datatype of tensor: {tensor.dtype}\")\n",
    "print(f\"Device tensor is stored on: {tensor.device}\")"
   ],
   "id": "597cdc427527d5c8",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of tensor: torch.Size([3, 4])\n",
      "Datatype of tensor: torch.float32\n",
      "Device tensor is stored on: cpu\n"
     ]
    }
   ],
   "execution_count": 6
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-10T14:20:40.272062Z",
     "start_time": "2024-10-10T14:20:40.269761Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# We move our tensor to the GPU if available\n",
    "if torch.cuda.is_available():\n",
    "    tensor = tensor.to(\"cuda\")"
   ],
   "id": "43195b454cf864bf",
   "outputs": [],
   "execution_count": 7
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-10T14:21:37.203185Z",
     "start_time": "2024-10-10T14:21:37.197947Z"
    }
   },
   "cell_type": "code",
   "source": [
    "tensor = torch.ones(4, 4)\n",
    "print(f\"First row: {tensor[0]}\")\n",
    "print(f\"First column: {tensor[:, 0]}\")\n",
    "print(f\"Last column: {tensor[..., -1]}\")\n",
    "tensor[:,1] = 0\n",
    "print(tensor)"
   ],
   "id": "8166ba8852ca1125",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First row: tensor([1., 1., 1., 1.])\n",
      "First column: tensor([1., 1., 1., 1.])\n",
      "Last column: tensor([1., 1., 1., 1.])\n",
      "tensor([[1., 0., 1., 1.],\n",
      "        [1., 0., 1., 1.],\n",
      "        [1., 0., 1., 1.],\n",
      "        [1., 0., 1., 1.]])\n"
     ]
    }
   ],
   "execution_count": 8
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-10T14:22:13.480323Z",
     "start_time": "2024-10-10T14:22:13.475815Z"
    }
   },
   "cell_type": "code",
   "source": [
    "t1 = torch.cat([tensor, tensor, tensor], dim=1)\n",
    "print(t1)"
   ],
   "id": "9cae3b07f23a3c82",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1., 0., 1., 1., 1., 0., 1., 1., 1., 0., 1., 1.],\n",
      "        [1., 0., 1., 1., 1., 0., 1., 1., 1., 0., 1., 1.],\n",
      "        [1., 0., 1., 1., 1., 0., 1., 1., 1., 0., 1., 1.],\n",
      "        [1., 0., 1., 1., 1., 0., 1., 1., 1., 0., 1., 1.]])\n"
     ]
    }
   ],
   "execution_count": 9
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-10T14:23:19.849240Z",
     "start_time": "2024-10-10T14:23:19.843239Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# This computes the matrix multiplication between two tensors. y1, y2, y3 will have the same value\n",
    "# ``tensor.T`` returns the transpose of a tensor\n",
    "y1 = tensor @ tensor.T\n",
    "y2 = tensor.matmul(tensor.T)\n",
    "\n",
    "y3 = torch.rand_like(y1)\n",
    "torch.matmul(tensor, tensor.T, out=y3)\n",
    "\n",
    "\n",
    "# This computes the element-wise product. z1, z2, z3 will have the same value\n",
    "z1 = tensor * tensor\n",
    "z2 = tensor.mul(tensor)\n",
    "\n",
    "z3 = torch.rand_like(tensor)\n",
    "torch.mul(tensor, tensor, out=z3)"
   ],
   "id": "1bda397ea76cc70a",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 0., 1., 1.],\n",
       "        [1., 0., 1., 1.],\n",
       "        [1., 0., 1., 1.],\n",
       "        [1., 0., 1., 1.]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 10
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-10T14:24:17.802494Z",
     "start_time": "2024-10-10T14:24:17.799747Z"
    }
   },
   "cell_type": "code",
   "source": [
    "agg = tensor.sum()\n",
    "agg_item = agg.item()\n",
    "print(agg_item, type(agg_item))"
   ],
   "id": "6731eaa0a7316b10",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12.0 <class 'float'>\n"
     ]
    }
   ],
   "execution_count": 11
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-10T14:26:15.344581Z",
     "start_time": "2024-10-10T14:26:15.340484Z"
    }
   },
   "cell_type": "code",
   "source": [
    "print(f\"{tensor} \\n\")\n",
    "tensor.add_(5)\n",
    "print(tensor)"
   ],
   "id": "1b2724c7acd7e97b",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1., 0., 1., 1.],\n",
      "        [1., 0., 1., 1.],\n",
      "        [1., 0., 1., 1.],\n",
      "        [1., 0., 1., 1.]]) \n",
      "\n",
      "tensor([[6., 5., 6., 6.],\n",
      "        [6., 5., 6., 6.],\n",
      "        [6., 5., 6., 6.],\n",
      "        [6., 5., 6., 6.]])\n"
     ]
    }
   ],
   "execution_count": 12
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-10T14:49:14.638138Z",
     "start_time": "2024-10-10T14:49:14.635001Z"
    }
   },
   "cell_type": "code",
   "source": [
    "t = torch.ones(5)\n",
    "print(f\"t: {t}\")\n",
    "n = t.numpy()\n",
    "print(f\"n: {n}\")"
   ],
   "id": "b0077b2339238804",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "t: tensor([1., 1., 1., 1., 1.])\n",
      "n: [1. 1. 1. 1. 1.]\n"
     ]
    }
   ],
   "execution_count": 13
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-10T14:50:07.100450Z",
     "start_time": "2024-10-10T14:50:07.095749Z"
    }
   },
   "cell_type": "code",
   "source": [
    "t.add_(1)\n",
    "print(f\"t: {t}\")\n",
    "print(f\"n: {n}\")"
   ],
   "id": "ab379c9ec55b6774",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "t: tensor([2., 2., 2., 2., 2.])\n",
      "n: [2. 2. 2. 2. 2.]\n"
     ]
    }
   ],
   "execution_count": 14
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-10T14:50:28.830414Z",
     "start_time": "2024-10-10T14:50:28.828164Z"
    }
   },
   "cell_type": "code",
   "source": [
    "n = np.ones(5)\n",
    "t = torch.from_numpy(n)"
   ],
   "id": "ccc1acdb0f07dfb8",
   "outputs": [],
   "execution_count": 15
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-10T14:50:44.000786Z",
     "start_time": "2024-10-10T14:50:43.997848Z"
    }
   },
   "cell_type": "code",
   "source": [
    "np.add(n, 1, out=n)\n",
    "print(f\"t: {t}\")\n",
    "print(f\"n: {n}\")"
   ],
   "id": "22fa25908f1b5146",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "t: tensor([2., 2., 2., 2., 2.], dtype=torch.float64)\n",
      "n: [2. 2. 2. 2. 2.]\n"
     ]
    }
   ],
   "execution_count": 16
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Datasets & DataLoaders",
   "id": "66308f4e62082a02"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T12:34:09.422945Z",
     "start_time": "2024-10-11T12:34:06.086215Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import torch\n",
    "from torch.utils.data import Dataset\n",
    "from torchvision import datasets\n",
    "from torchvision.transforms import ToTensor\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "training_data = datasets.FashionMNIST(\n",
    "    root=\"data\",\n",
    "    train=True,\n",
    "    download=True,\n",
    "    transform=ToTensor()\n",
    ")\n",
    "\n",
    "test_data = datasets.FashionMNIST(\n",
    "    root=\"data\",\n",
    "    train=False,\n",
    "    download=True,\n",
    "    transform=ToTensor()\n",
    ")"
   ],
   "id": "e31f92a0ac210e85",
   "outputs": [],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T12:34:13.196282Z",
     "start_time": "2024-10-11T12:34:12.985658Z"
    }
   },
   "cell_type": "code",
   "source": [
    "labels_map = {\n",
    "    0: \"T-Shirt\",\n",
    "    1: \"Trouser\",\n",
    "    2: \"Pullover\",\n",
    "    3: \"Dress\",\n",
    "    4: \"Coat\",\n",
    "    5: \"Sandal\",\n",
    "    6: \"Shirt\",\n",
    "    7: \"Sneaker\",\n",
    "    8: \"Bag\",\n",
    "    9: \"Ankle Boot\",\n",
    "}\n",
    "figure = plt.figure(figsize=(8, 8))\n",
    "cols, rows = 3, 3\n",
    "for i in range(1, cols * rows + 1):\n",
    "    sample_idx = torch.randint(len(training_data), size=(1,)).item()\n",
    "    img, label = training_data[sample_idx]\n",
    "    figure.add_subplot(rows, cols, i)\n",
    "    plt.title(labels_map[label])\n",
    "    plt.axis(\"off\")\n",
    "    plt.imshow(img.squeeze(), cmap=\"gray\")\n",
    "plt.show()"
   ],
   "id": "645427837456c6c6",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 800x800 with 9 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAn4AAAKQCAYAAAABnneSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABdOElEQVR4nO3deXRV5dX48Z3hZp5DRgUCYQgySBEHUCZRUCaLWEUcqQO2tQ4vvoLDT9raiqI4rCq2bxUc6oC1UqkoiDLpC68ELSCigmAimoRAmELInPP7o4vUmGc/5F5vGPJ8P2uxluxz9znnntxzz/aQvU+I53meAAAAoM0LPdY7AAAAgKODwg8AAMARFH4AAACOoPADAABwBIUfAACAIyj8AAAAHEHhBwAA4AgKPwAAAEdQ+AEAADiCwi+IPvroIxk/frx06NBBIiMjJSMjQwYMGCBTp05tfE1OTo6MGTPmiOtasWKFhISEyIoVK1q07Zdfflkef/zxAPccOD6EhIS06I/tvFiyZImMGDFCsrOzJTIyUrKzs2Xo0KHy4IMPNtvWzTfffMR9eu655yQkJEQKCgpa9B7mzJkjzz33XIteC5xIDp8L3/+TlpYmQ4cOlbfeeutY7x5aKPxY70BbsWjRIhk3bpwMHTpUZs2aJVlZWVJcXCzr1q2TV199VWbPnu3X+vr16ydr1qyRU045pUWvf/nll2XTpk1y2223BbD3wPFhzZo1Tf5+//33y/Lly2XZsmVN4tp58ac//Ul+8YtfyIQJE+TJJ5+UlJQU2bFjh6xevVpef/11mT59ut/7NHr0aFmzZo1kZWW16PVz5syRdu3aybXXXuv3toATwbx58yQvL088z5OSkhJ58sknZezYsbJw4UIZO3bssd49HAGFX5DMmjVLOnXqJEuWLJHw8P8c1okTJ8qsWbP8Xl9CQoKcddZZR3zdoUOHJCYmxu/1A8ejH37m09LSJDQ0tEXngojIzJkzZfDgwfL66683iV911VXS0NAQ0D6lpaVJWlraEV/HuQhX9OrVS/r379/49wsuuECSk5PllVdeofA7AfBPvUFSVlYm7dq1a1L0HRYa2vwwL168WPr16yfR0dGSl5cnc+fObbLc9E+91157rcTFxcmnn34qI0aMkPj4eBk+fLgMHTpUFi1aJIWFhU1uwQOuKSsrU+/Mmc5DEZEXX3xRevToITExMXLqqac2+ycr0z/1Dh06VHr16iWrVq2SgQMHSkxMjPz85z+XnJwc+eyzz2TlypWN52FOTk6w3h5wXIqKipKIiAjx+XyNsd/+9rdy5plnSkpKiiQkJEi/fv3k2WefFc/zmuRWV1fL1KlTJTMzU2JiYmTw4MHy8ccfS05ODnfNWwl3/IJkwIAB8swzz8gtt9wiV1xxhfTr16/JSfB9GzZskKlTp8r06dMlIyNDnnnmGbnuuuukS5cuMnjwYOt2ampqZNy4cTJlyhSZPn261NXVycknnyw33nijbNu2TRYsWNAabw84IQwYMED+/ve/y29+8xsZP3689OrVS8LCwtTXL1q0SPLz8+V3v/udxMXFyaxZs2T8+PHy5ZdfSufOna3bKi4uliuvvFLuvPNOeeCBByQ0NFSmTZsml1xyiSQmJsqcOXNERCQyMjKo7xE41urr66Wurk48z5OdO3fKww8/LBUVFTJp0qTG1xQUFMiUKVOkQ4cOIiLyf//3f/LrX/9avvvuO7nvvvsaXzd58mSZP3++3HnnnXLuuefK5s2bZfz48XLgwIGj/r6c4SEodu/e7Z1zzjmeiHgi4vl8Pm/gwIHezJkzvfLy8sbXdezY0YuKivIKCwsbY5WVlV5KSoo3ZcqUxtjy5cs9EfGWL1/eGLvmmms8EfHmzp3bbPujR4/2Onbs2CrvDThWrrnmGi82NrbFr//qq6+8Xr16NZ6H0dHR3vDhw70nn3zSq6mpafJaEfEyMjK8AwcONMZKSkq80NBQb+bMmY2xefPmeSLiff31142xIUOGeCLivf/++832oWfPnt6QIUNa/iaBE8Thc+GHfyIjI705c+aoefX19V5tba33u9/9zktNTfUaGho8z/O8zz77zBMRb9q0aU1e/8orr3gi4l1zzTWt+XacxR2/IElNTZUPPvhA1q1bJ++//76sW7dOVqxYIXfddZf8+c9/lvz8fGnXrp2IiPTt27fx/4JE/n2bvFu3blJYWNiibU2YMKFV3gNwIvA8T+rr65vEDv+KRW5urmzYsEE+/PBDWbFihaxbt05Wrlwp77//vsybN08+/PBDiYqKaswbNmyYxMfHN/49IyND0tPTW3QuJicny7nnnhukdwWcOF544QXp0aOHiIjs3r1bFixYIL/61a+kvr6+sVN+2bJl8sADD0h+fn6zu3elpaWSkZEhK1euFBGRSy+9tMnySy65RK666qqj8E7cxO/4BVn//v1l2rRp8re//U2Kiork9ttvl4KCgiYNHqmpqc3yIiMjpbKy8ojrj4mJkYSEhKDuM3Aief7558Xn8zX5832hoaEyePBgue+++2ThwoVSVFQkl112mXz88cfNfpf2x5yLLe3yBdqaHj16SP/+/aV///5ywQUXyJ///GcZMWKE3HnnnbJv3z5Zu3atjBgxQkRE/vKXv8j//u//Sn5+vtxzzz0iIo3nV1lZmYj8+3+4vi88PNx4biI4KPxakc/nkxkzZoiIyKZNm4KyTpo24LqxY8dKfn5+kz82sbGxctddd4lI8M5DEc5F4Pv69OkjlZWVsmXLFnn11VfF5/PJW2+9JZdeeqkMHDiwSRfwYYeLu507dzaJ19XVNRaFCD7+qTdIiouLjXcAPv/8cxERyc7ObtXtt/QuBXCiS01NVe8GHOvzUIRzEW5av369iPx7/FFISIiEh4c3aayqrKyUF198sUnO4WbG+fPnS79+/Rrjr7/+utTV1bX+TjuKwi9IRo4cKSeffLKMHTtW8vLypKGhQdavXy+zZ8+WuLg4ufXWW1t1+71795Y33nhDnn76aTnttNMkNDTU+H9YQFvWs2dPGT58uFx44YWSm5srVVVV8tFHH8ns2bMlIyNDrrvuulbfh969e8urr74q8+fPl86dO0tUVJT07t271bcLHC2bNm1qLMzKysrkjTfekKVLl8r48eOlU6dOMnr0aHn00Udl0qRJcuONN0pZWZk88sgjzTrce/bsKZdffrnMnj1bwsLC5Nxzz5XPPvtMZs+eLYmJieoIJvw4FH5Bcu+998qbb74pjz32mBQXF0t1dbVkZWXJeeedJ3fddVfjL8K2lltvvVU+++wzufvuu2X//v3ieV6zeUlAW/fggw/KkiVL5A9/+IOUlJRIXV2dtG/fXiZNmiT33HPPUfm9vN/+9rdSXFwsN9xwg5SXl0vHjh1b/Lg34EQwefLkxv9OTEyUTp06yaOPPiq//OUvRUTk3HPPlblz58pDDz0kY8eOlZNOOkluuOEGSU9Pb/Y/X/PmzZOsrCx59tln5bHHHpO+ffvKa6+9JhdccIEkJSUdzbfljBCP6gAAABwnVq9eLWeffba89NJLTWYDIjgo/AAAwDGxdOlSWbNmjZx22mkSHR0tGzZskAcffFASExNl48aNTcYvITj4p14AAHBMJCQkyLvvviuPP/64lJeXS7t27eTCCy+UmTNnUvS1Eu74AQAAOIKWGQAAAEdQ+AEAADiCwg8AAMARFH4AAACOaHFXb1t7LuVNN91kjP/wYdHf16dPH2N89erVas727duN8c6dO6s5GzduNMYHDhzo974tWLBAzfnrX/+qLnPF8djb1NbONY1tKn9DQ4MxnpycrObMnDnTGE9ISFBzoqOjjfHq6mo154ePnTps0aJFao7m+4+0+iHts2n7zB6Pn+fDjsd9c+Vcg1uOdK5xxw8AAMARFH4AAACOoPADAABwBIUfAACAI1r85I629kuwL7zwgjG+YcMGNSc3N9cYr6ioUHNiY2ODlrNz5041R2NrCPnv//5vY1xrLmmL+IVz/2gNGUer4WDu3LnqsgkTJhjjhYWFas7bb79tjF977bVqjtYAdrR+bramGO1Y2/ZNa6QJNs414OiguQMAAAAiQuEHAADgDAo/AAAAR1D4AQAAOILCDwAAwBEUfgAAAI5o8bN62xptJENcXJyas23bNr9zDh486Ff8SOvzdzu2Zw+ff/75xrhL41xcFsholmCO/rCNGpo8ebIxrn1mRUQ++eQTY/yLL75Qc7RxHnv27FFztPEw+fn5ao72PO+FCxeqOe+//74xHsjPgLElAA7jjh8AAIAjKPwAAAAcQeEHAADgCAo/AAAAR1D4AQAAOKJNd/XGxsaqywLp6g2mQLYTSPewLedovVccn7TuXVtXb1RUlDH+1FNPqTndunUzxrOystSciIgIY3zr1q1qTmpqqjF+6qmnqjn9+vUzxouKitSc5cuXG+PnnHOOmvPTn/7UGB89erSas2vXLmN8w4YNas5NN91kjNs6gbWO3yM96B3AiYk7fgAAAI6g8AMAAHAEhR8AAIAjKPwAAAAcQeEHAADgCAo/AAAAR7TpcS62cSU7d+40xktKSoK6nWDSRraIiGzcuNEY18bWiIj06dPnR+8TTlyBjOt4+eWXjfGf/OQnas7u3buNce0cFBHx+XzGeHV1tZqjnR+PPvqomnPttdca47m5uWqONhqlqqpKzSksLDTGtVEqIiJhYWHG+PDhw9Wct99+2xgfNWqUmsPYFsAt3PEDAABwBIUfAACAIyj8AAAAHEHhBwAA4AgKPwAAAEe06a7egQMHqsu2bdvm9/oyMzP9zrF14vrL1j2sLbO9z6PVjYwTy+23364u69GjhzFeUFCg5kRGRhrj4eH61099fb0xbjsHtfWdffbZas5JJ51kjNs6jgcMGGCM19TUqDlal3JdXZ2aox2Dr7/+Ws1p3769MX7hhReqOe+88466rK0LDTXf+9A6t0X0z2Dv3r3VnF27dhnjUVFRak5ERIS6TKN9Bm0d59p5o3WVi+jd6LbtBJPtu0Pbb+18EtH329bxrq3P9jPVPm+27w5tfZ988omacyTc8QMAAHAEhR8AAIAjKPwAAAAcQeEHAADgCAo/AAAAR1D4AQAAOKJNj3PJyMg41rtw1MTGxhrjtrEU2jiXPn36qDkbN270b8dwwrn44ovVZZWVlca4NrJFRB/9YBuZoY09OHTokN/bsX2etfdjGxehjWiyjb/Q2MZFaGxjPrRjetlll6k5Lo9zCUROTo4xbhs1tHfvXr+3c+DAAWPcNoZLG/1hGxuknQO28Sfp6enGuO2zWVZWpi7TpKamGuO275vy8nJjXHufIiLx8fHGeLt27dQcbURTcXGxmqN9d6SkpKg52gitoqIiNedIuOMHAADgCAo/AAAAR1D4AQAAOILCDwAAwBEUfgAAAI5wtqtX63atqKhord1pVVqn17Zt29ScgQMHGuO5ublqDl29bV/nzp3VZSUlJca41lErYu8o1GgdsoFsx9YBqD2c3dahq3Uc2zp0tW5bW462TOsmFNGPga3j1GW2znKNdl2JiYlRcxITE41x289F+znv379fzdHeT9euXdUc7fyora1Vc7QO3ejoaDXn9NNPN8Ztn2etC9bWJa0dU60b27YPtmOtddXm5eWpOe3btzfGbdMKunfvbowvWbJEzTkS7vgBAAA4gsIPAADAERR+AAAAjqDwAwAAcASFHwAAgCMo/AAAABzBOBc/xMbG+p2jjb+wPWg7mAIZT2N7qP2CBQt+zO7gODJ69GhjPJCRKbaHme/atcsYr6mpseydmW38iUYbvxLo+jTJycnqMu247dmzR83Rfg7V1dVqTni4+Su9Y8eOag78o41tOemkk9Sc0tJSY1wbVyIikp6ebox36dJFzdHGudjGhWjXCNuomaysLGP8u+++U3O092rbN02HDh3UZQkJCca49jMQEdmxY4cxnpqaqub85Cc/UZdpiouLjfGCggI1R3uv2rneEtzxAwAAcASFHwAAgCMo/AAAABxB4QcAAOAICj8AAABHtOmuXlvnbCBdvRpb9/C2bduCth1bB1ggtG4u2/tB2zF48GBjXHv4uIhIfX29Ma49hF5EpLy83BjXOl1ttO3b2LYTSGectj6tg19EJCwszBi3dRUnJSUZ47ZjvW/fPmNc66wW0bsgDxw4oOa4TPsMap9zEZH27dsb4ykpKWpOYWGhMf7tt9+qOdo1T/ssiehdyt98843fOTk5OWqO1sGudQiLiERGRhrjtu5hrUPWdgzOOussY1zrkhYR+frrr43xsrIyNUf7HJxxxhlqTufOnY1x2/s5Eu74AQAAOILCDwAAwBEUfgAAAI6g8AMAAHAEhR8AAIAjKPwAAAAc0abHudhGtmgjSwIZ86KNRTlRMc7FDQMGDDDGa2pq1BxtmW3sgfag88rKSjVHW582FiXQnJCQEHWZRlufz+dTcyIiIoxx23eHNuYiNFT//3XtQfRpaWlqzumnn26Mv//++2qOy7Sff2xsrJqjjUiynTfa97Dtc7Z7925j3DbORzs/tTEiIiJVVVXGuG0EjDaaRRvzIiKyd+9eYzwqKkrN6dOnj7pMox0329iYrl27GuN5eXlqjjaSbcuWLWqONjrHNq7uSLjjBwAA4AgKPwAAAEdQ+AEAADiCwg8AAMARFH4AAACOaNNdvVoHjYi9A8tftnVpXXs/piPnWNI6vbZv336U9wQ/lvaQ8V69eqk5PXv2NMZXrVql5nTo0MEY17rVRET2799vjNs6dLVuV1vnZCA5Glu3pdYNbcvRHvaekpKi5midk+Xl5WrOmjVr1GVoLiEhwRivrq5Wc9q3b+/3drSfv/YzFhFJSkoyxtu1a6fmaJ3gBw4cUHOio6ONce1cF9Gvx1qHsIhI//79/VqXiN4ha+sE1q7H2vZFRLZt22aMa9+rInp3v23fMjMzjfHk5GQ150i44wcAAOAICj8AAABHUPgBAAA4gsIPAADAERR+AAAAjqDwAwAAcESbHueiPeTa5miNZrHtm+3B7Rqtvd3W9q7tQ0lJiZqTm5trjDPO5cRzzTXXGOPayAERkVNOOcUYX79+vZqzceNGYzw8/Pj9+vE8T12m7bdtBIw26kMbDSKij9no3bu3mhPM8TQw08ac2MYT7d271xhPS0tTc7QRLLbrgza2xzZqRhvRpX3+RPTrSnFxsZpTW1trjNvG06xbt05dpsnJyTHGExMT1ZxDhw4Z45s3b1ZztJ93t27d1BxtHJVt3JI22mrXrl1qzpFwxw8AAMARFH4AAACOoPADAABwBIUfAACAIyj8AAAAHHH8ttUdI4F01NpoXcLB3o5m586dQV1fIB3MOLHU1NSoy2zduxrtHNC61UTsXbXBFBISYoxr3XeB0rptte2LiMTHx/u9Hbp3W5/Wiav9jEVEfD6fMV5YWKjmaJ/BpKQkfecUWteqiMi//vUvY9zWbfuTn/zEGLd19Wrdw7buVK1D1zatQutG1jqrRfTrcUpKipqjdQnbvte07mrb+9GOdc+ePdWcI+GOHwAAgCMo/AAAABxB4QcAAOAICj8AAABHUPgBAAA4gsIPAADAEW16nIttZIo2YsLWVl1SUmKMDxw4UM05WuNPtO1s375dzdGOgW2ftfe6YMECy97hRGIbZVJfX39UtnOs2casaMcgkPdjG52jPdQex5Y2YqSurs7vdQ0ePNjvnG+//VZdpn0GzzzzTDVHGwG0ceNGNSc6OtoY/8UvfqHmaONuUlNT1Rzterxq1So1Rxspo41FERFp166dMb5jxw41RxuRo415EdGvrbZrbpcuXYzx7777Ts05Eu74AQAAOILCDwAAwBEUfgAAAI6g8AMAAHAEhR8AAIAj2nRXb0ZGhrps586dxritQ1fLsW0nkK5erds2kBztwdgi9q5nje29om3QuvwCpXWuBns7tk5cf2kdiCJ6t62tq9fzPGM82McArU/rpuzatauaEx5uvtRqHcIiIj6fzxg/9dRT1RxtioM2kUJEpEePHsZ4t27d1Jzq6mpjPDMzU80pKyszxvft26fmFBQUGOO267S23wcOHFBzYmJijHHbNVJ7r7ZrvvYztU0T0aYI/OMf/1Bzbr75ZnWZCHf8AAAAnEHhBwAA4AgKPwAAAEdQ+AEAADiCwg8AAMARFH4AAACOaNPjXGz69OljjGsjW460zF+29m1tNIutTXzDhg3GeG5urpqjLTtaxwBu0EZZaCNOROyjUfxlG82i7YNtNIy2zLYd2zKcWIqKiozxk046Sc3RPs8ffvihmqONjenQoYOa07FjR2PcNs5l06ZNxrg2ekREJD093RifP3++mqONTCktLVVzEhMTjfH27durOZWVlcb4e++9p+bU1dUZ47ZrbnJysjH+5Zdfqjnl5eXGuG2sk/azS0hIUHOOhG8jAAAAR1D4AQAAOILCDwAAwBEUfgAAAI6g8AMAAHBEm+7qvfrqq9VlN910kzF+/vnnB3UfbN27wVyX1n1ke8j0u+++a4yPHDnSvx1Dm2LraLV14moiIyONcdvD2QPpnNU64wJ5P9qD0UX0LuWamhq/t2Nj2wccO9pnUOvCFRHZs2ePMd6vXz81p1evXn5v53//93+N8aysLDVH60betm2bmqN1tO7du1fN0c5P2zldVlZmjG/evFnN0bpg09LS1Byt69r2HaV16B46dEjN0dTW1qrLqqqqjPGoqCi/t3MYd/wAAAAcQeEHAADgCAo/AAAAR1D4AQAAOILCDwAAwBEUfgAAAI5o0+NcbAIZsxJITkZGhjG+c+dOv9cVyHa0uIjIjBkzgroPaBts4080tpEMhYWFxrg25kVEH69gG4ui7YPtAejBfq/HOgf+CeQzo+UkJiaqOdr6Nm3apOYkJCQY49o4IRGRmJgYY3z37t1qTnFxsTHu8/nUHG3MSs+ePdWcjz/+2BiPj49Xc7Kzs41x23HTzunY2Fg1RxuZUl1dreZox0f77hLRPzu2cS51dXXqskDxzQIAAOAICj8AAABHUPgBAAA4gsIPAADAERR+AAAAjnC2q7eiosIYz8zM9DvH5tRTTzXGV69e7fe6bLQuYVtXr+3B3YA/AumOtXXoBnM7ti47TX19vd85tn0L5L1qHZqBCPa+tRW27l2N1iGbk5Oj5mgTIU466SQ1R+s0tXV5ah2/tvepdSPbOk21Y1BSUqLmaOuzdRzv27fPGLe9H+0abtuOxjZ5QGPrutbYfqbacfsxXf/c8QMAAHAEhR8AAIAjKPwAAAAcQeEHAADgCAo/AAAAR1D4AQAAOMLZcS7amJNt27YFdTtaG3+wxcXFGePamBcRkT59+hjjCxYsCMo+wR2BjMWw5WgjWGw5gYwl0ca22Nal7YNtbEwg42ECOaYIDtuojFGjRhnjlZWVas7evXuN8erqajVHG7e1f/9+Ncfn8xnjCQkJak4gtH0rKipSc3Jzc/3ejnb9tJ1P2jGwndPa+myjWQL5vglkNIu2b4xzAQAAwBFR+AEAADiCwg8AAMARFH4AAACOoPADAABwBF29QRBI564tJzY21u/1BZIzcOBAv3PQ9gXSrWbL6dy5szFeWFio5mgPLdc69kQC65zVuvZsD023de9qAjmmVVVVfufAP/PnzzfGhw0bpuakpaUZ4x9++KGao62vtLRUzdGuEbYO3ZNOOkldptHOG9s5oHWnnnPOOWrOP/7xD2Pcdi1MTEw0xrUpFiIiZWVl6rJg0r6LtGMjondx19TUqDnfffedMb5mzRrL3tlxxw8AAMARFH4AAACOoPADAABwBIUfAACAIyj8AAAAHEHhBwAA4Ahnx7ls27bNGB8xYoSas3PnTmO8oqJCzdHazm3t6LZlGq0l3raujRs3+r0dtH2BjB6x2bNnjzFuewC6NsokIiJCzdEeWm4b8xISEmKM20a2aDm249bQ0OD3doIp2D/TtuLaa681xlNTU9WcmJgYY7xbt25qzu7du43x8vJyNSc6Otqv7YuIpKenG+O2kSmBfJ61nLy8PDXnscceU5fh6OKOHwAAgCMo/AAAABxB4QcAAOAICj8AAABHUPgBAAA4wtmu3q+++soYj42NVXO0ZbacjIwMY3zDhg1+b8fG1rUVzBy0fVrHnkhg3aFat62N1u0aSKeh7f34uy4R/f3YHmofyHGzdT0jOCorK43xb7/91u91bdmy5cfuzgnvnXfeOda7gBbgjh8AAIAjKPwAAAAcQeEHAADgCAo/AAAAR1D4AQAAOILCDwAAwBHMC/iBbdu2qcsqKir8Xt/27duN8Z07d/q9nUDGxsTFxVn2Dmh9Pp/PGK+trVVz6uvrjXHbiBNtzIo2GkYkuKNetH0OdDs1NTV+5wDAkXDHDwAAwBEUfgAAAI6g8AMAAHAEhR8AAIAjKPwAAAAc4WxXb3x8vDFu69w9ePCg3zkbNmwwxm3dtlr3rm07WpdwIO8HCKYvv/zSGE9KSlJztE7choYGNUfrqvU8T985hW07/m7ftg+2fSstLfV7HwLpHg7k+AA4cXHHDwAAwBEUfgAAAI6g8AMAAHAEhR8AAIAjKPwAAAAcQeEHAADgCGfHuaxevdoYv+qqq9Sc3Nxcv7ej5Wjbt7GNX8nIyPArHug+AP7SxhMlJCSoOXV1dcZ4cnKymqONgAlkxEmwaaNZbPt2/vnnG+Ohofr/r2tjaGw5jHMB3MIdPwAAAEdQ+AEAADiCwg8AAMARFH4AAACOoPADAABwhLNdvRUVFcb41VdfreaMHz/eGH/44YfVHFsnrkbrxO3cubOaExcX5/f2FyxY4N+OwQnB7vL89NNPjfGqqio1p6amxhi3dadqOVpcRO8eDnYnsNZtGxMTo+bU1tb6ta5Atg/APdzxAwAAcASFHwAAgCMo/AAAABxB4QcAAOAICj8AAABHUPgBAAA4IsTjCd0AAABO4I4fAACAIyj8AAAAHEHhBwAA4AgKPwAAAEdQ+AEAADiCwg8AAMARFH4AAACOoPADAABwBIUfAACAIyj8AAAAHEHhBwAA4AgKPwAAAEdQ+AEAADiCwg8AAMARFH4AAACOoPALoo8++kjGjx8vHTp0kMjISMnIyJABAwbI1KlTG1+Tk5MjY8aMOeK6VqxYISEhIbJixYoWbfvll1+Wxx9/PMA9B44fLTmPjpaCggIJCQmR5557zu9cf89h4Fh77rnnJCQkpPFPVFSUZGZmyrBhw2TmzJlSWlp6rHcRQUDhFySLFi2SgQMHyoEDB2TWrFny7rvvyhNPPCFnn322zJ8/3+/19evXT9asWSP9+vVr0esp/NAWBPs8AuC/efPmyZo1a2Tp0qXy1FNPSd++feWhhx6SHj16yHvvvXesdw8/Uvix3oG2YtasWdKpUydZsmSJhIf/57BOnDhRZs2a5ff6EhIS5Kyzzjri6w4dOiQxMTF+rx84HgX7PALgv169ekn//v0b/z5hwgS5/fbb5ZxzzpGLL75Ytm7dKhkZGcZcrknHP+74BUlZWZm0a9euycXqsNDQ5od58eLF0q9fP4mOjpa8vDyZO3duk+Wmfya69tprJS4uTj799FMZMWKExMfHy/Dhw2Xo0KGyaNEiKSwsbHKbHjjRtPQ8mj9/vowYMUKysrIkOjpaevToIdOnT5eKioomOYfPma+++kpGjRolcXFx0r59e5k6dapUV1c3eW1RUZFceumlEh8fL4mJiXLZZZdJSUlJs/1Yt26dTJw4UXJyciQ6OlpycnLk8ssvl8LCwiAdBeD406FDB5k9e7aUl5fLn//8ZxHRr0kiIjU1NfL73/9e8vLyJDIyUtLS0mTy5Mmya9euJutdtmyZDB06VFJTUyU6Olo6dOggEyZMkEOHDjW+5umnn5ZTTz1V4uLiJD4+XvLy8uTuu+8+em++jeGOX5AMGDBAnnnmGbnlllvkiiuukH79+onP5zO+dsOGDTJ16lSZPn26ZGRkyDPPPCPXXXeddOnSRQYPHmzdTk1NjYwbN06mTJki06dPl7q6Ojn55JPlxhtvlG3btsmCBQta4+0BR0VLz6OtW7fKqFGj5LbbbpPY2Fj54osv5KGHHpK1a9fKsmXLmry2trZWxo0bJ9ddd51MnTpVVq1aJffff78kJibKfffdJyIilZWVct5550lRUZHMnDlTunXrJosWLZLLLrus2bYLCgqke/fuMnHiRElJSZHi4mJ5+umn5fTTT5fNmzdLu3btWufgAMfYqFGjJCwsTFatWtUYM12TGhoa5KKLLpIPPvhA7rzzThk4cKAUFhbKjBkzZOjQobJu3TqJjo6WgoICGT16tAwaNEjmzp0rSUlJ8t1338nixYulpqZGYmJi5NVXX5Vf/vKX8utf/1oeeeQRCQ0Nla+++ko2b958DI/ECc5DUOzevds755xzPBHxRMTz+XzewIEDvZkzZ3rl5eWNr+vYsaMXFRXlFRYWNsYqKyu9lJQUb8qUKY2x5cuXeyLiLV++vDF2zTXXeCLizZ07t9n2R48e7XXs2LFV3htwtLT0PPq+hoYGr7a21lu5cqUnIt6GDRsalx0+Z1577bUmOaNGjfK6d+/e+Penn37aExHvzTffbPK6G264wRMRb968eeo+19XVeQcPHvRiY2O9J554ojFuOoeB49m8efM8EfHy8/PV12RkZHg9evTwPE+/Jr3yyiueiHh///vfm8Tz8/M9EfHmzJnjeZ7nvf76656IeOvXr1e3d/PNN3tJSUmBviUY8E+9QZKamioffPCB5Ofny4MPPigXXXSRbNmyRe666y7p3bu37N69u/G1ffv2lQ4dOjT+PSoqSrp169bifyqaMGFC0PcfOB609Dzavn27TJo0STIzMyUsLEx8Pp8MGTJEREQ+//zzJusMCQmRsWPHNon16dOnyfm2fPlyiY+Pl3HjxjV53aRJk5rt48GDB2XatGnSpUsXCQ8Pl/DwcImLi5OKiopm2wbaGs/zmsV+eE166623JCkpScaOHSt1dXWNf/r27SuZmZmNv8LUt29fiYiIkBtvvFGef/552b59e7N1n3HGGbJv3z65/PLL5c0332xyLUVgKPyCrH///jJt2jT529/+JkVFRXL77bdLQUFBk19MT01NbZYXGRkplZWVR1x/TEyMJCQkBHWfgeON7Tw6ePCgDBo0SD766CP5/e9/LytWrJD8/Hx54403RESanUcxMTESFRXVJBYZGSlVVVWNfy8rKzP+snpmZmaz2KRJk+TJJ5+U66+/XpYsWSJr166V/Px8SUtLa9E5DJyoKioqpKysTLKzsxtjpmvSzp07Zd++fRIRESE+n6/Jn5KSksbiLTc3V9577z1JT0+XX/3qV5Kbmyu5ubnyxBNPNK7rqquukrlz50phYaFMmDBB0tPT5cwzz5SlS5cenTfdBvE7fq3I5/PJjBkz5LHHHpNNmzYFZZ00bcA1PzyPli1bJkVFRbJixYrGu3wiIvv27Qt4G6mpqbJ27dpm8R82d+zfv1/eeustmTFjhkyfPr0xXl1dLXv27Al4+8CJYNGiRVJfXy9Dhw5tjJmuSe3atZPU1FRZvHixcT3x8fGN/z1o0CAZNGiQ1NfXy7p16+SPf/yj3HbbbZKRkSETJ04UEZHJkyfL5MmTpaKiQlatWiUzZsyQMWPGyJYtW6Rjx47BfZMO4I5fkBQXFxvjh//p5/v/h9QaWnrHEDieteQ8OnyhiYyMbPKaw52GgRg2bJiUl5fLwoULm8RffvnlJn8PCQkRz/OabfuZZ56R+vr6gLcPHO+++eYbueOOOyQxMVGmTJlife2YMWOkrKxM6uvrpX///s3+dO/evVlOWFiYnHnmmfLUU0+JiMgnn3zS7DWxsbFy4YUXyj333CM1NTXy2WefBefNOYY7fkEycuRIOfnkk2Xs2LGSl5cnDQ0Nsn79epk9e7bExcXJrbfe2qrb7927t7zxxhvy9NNPy2mnnSahoaFN5jABJ4KWnEfZ2dmSnJwsN910k8yYMUN8Pp+89NJLsmHDhoC3e/XVV8tjjz0mV199tfzhD3+Qrl27yttvvy1Llixp8rqEhAQZPHiwPPzww9KuXTvJycmRlStXyrPPPitJSUk/8t0Dx4dNmzY1/l5eaWmpfPDBBzJv3jwJCwuTBQsWSFpamjV/4sSJ8tJLL8moUaPk1ltvlTPOOEN8Pp98++23snz5crnoootk/Pjx8qc//UmWLVsmo0ePlg4dOkhVVVXjaLPzzjtPRERuuOEGiY6OlrPPPluysrKkpKREZs6cKYmJiXL66ae3+rFoiyj8guTee++VN998Ux577DEpLi6W6upqycrKkvPOO0/uuusu6dGjR6tu/9Zbb5XPPvtM7r77btm/f794nmf8JVzgeNbS82jRokUydepUufLKKyU2NlYuuugimT9/foufdPNDMTExsmzZMrn11ltl+vTpEhISIiNGjJBXX31VBg4c2OS1L7/8stx6661y5513Sl1dnZx99tmydOlSGT169I9+/8DxYPLkySIiEhERIUlJSdKjRw+ZNm2aXH/99Ucs+kT+ffdu4cKF8sQTT8iLL74oM2fOlPDwcDn55JNlyJAh0rt3bxH5d3PHu+++KzNmzJCSkhKJi4uTXr16ycKFC2XEiBEi8u9/Cn7uuefktddek71790q7du3knHPOkRdeeKFF+4LmQjyqAwAAACfwO34AAACOoPADAABwBIUfAACAIyj8AAAAHEHhBwAA4AgKPwAAAEdQ+AEAADiixQOc29ozYnv16mWMX3rppWpOt27djPFnn31WzVmxYoUxXldXp+b88IHyh9keyebz+Yzxw0MwTS644AJj/L333lNz3n33Xb/37Xh2PI6xbGvnGiDCuXa8uuKKK4zx66+/Xs3RhpUfOnQoKPt02KpVq4zxV155Rc15+umng7oPJ6IjnWvc8QMAAHAEhR8AAIAjKPwAAAAcQeEHAADgCAo/AAAAR4R4LWy1OhG7n+Lj49Vl27ZtM8aff/55NUfrBO7cubOas337dmN8x44das6//vUvY/ynP/2pmlNaWmqM5+TkqDmrV682xvft26fmfPfdd8b4Cy+8oOYcz+g0BI4OzrXWd9ZZZxnjL730kpqjXYuSk5PVnMTERGM8IyPDsndmERER6rKtW7ca47t27VJzYmNjjfHJkyerORs2bFCXnYjo6gUAAICIUPgBAAA4g8IPAADAERR+AAAAjqDwAwAAcASFHwAAgCPa9DgXmxdffNEYLykpUXOqqqqM8SFDhqg51dXVxvgdd9yh5mit5e+8846as3PnTmO8qKhIzSkoKDDGbWNwFixYYIxrY2uOd4yYAI4OzrXgGD16tLps6tSpxrjP5/N7O7W1tX7npKamqsvCw8ON8T179qg52mcmLCxMzQkNNd/P2r9/v5rzyCOPGOPvvfeemnM8Y5wLAAAARITCDwAAwBkUfgAAAI6g8AMAAHAEhR8AAIAjzG02DqivrzfGo6Ki1Byt23bUqFFqTlpamjEeSBdsXFycumzLli3G+ObNm9Uc7UHbSUlJak5dXZ26DADQuqKjo9Vle/fuNcYTEhLUHG2Kg60TWJtwsWvXLjVHu3bExsaqOVpncWRkpJqjde9qEzZERJKTk9VlbRF3/AAAABxB4QcAAOAICj8AAABHUPgBAAA4gsIPAADAERR+AAAAjnB2nEtlZaUxbhtXoj2A+sMPP1RzbrnlFmO8Y8eOas6mTZuM8fbt26s5c+fONcZtD80uLS01xm1jY2pqatRlAIDW9frrr6vLcnJyjPGLL75YzdFGm9muhSEhIca47dqhjYDxPE/NCQsL82tdIvromrffflvN+dvf/qYua4u44wcAAOAICj8AAABHUPgBAAA4gsIPAADAERR+AAAAjnC2q7ehocEYj4qKUnN+8pOfGONaR62IyNVXX22M33vvvWrOxIkTjfHdu3erOfv27TPG09PT1Rzt4dy2h4ADAI5PjzzyiDE+btw4NUe75tmuNzExMca4rUO3trZWXabRpkhkZGSoOdo+PP74435vv63ijh8AAIAjKPwAAAAcQeEHAADgCAo/AAAAR1D4AQAAOILCDwAAwBHOjnPR2B7+fODAAWP87LPPVnMmTZpkjP/Xf/2XmvPf//3fxviLL76o5mRnZxvjhw4dUnNiY2ONcdvDubUcAEDrCw3V79doY8pef/11Nef222/3a10iIj6fzxivr69XczQRERHqMm2cS2pqqpozb948v/fBNdzxAwAAcASFHwAAgCMo/AAAABxB4QcAAOAICj8AAABHONvV2717d2N8zZo1ak5YWJgxHh0dreacfvrpxvi7776r5nTu3NkY37hxo5qjdVPZOrO+++47Y7xXr15+79u2bdvUHABAcHie53fOnj171GUhISHGeExMjN/7EEgnsK2rV9tOeLheutjeq0brlLa9nxMZd/wAAAAcQeEHAADgCAo/AAAAR1D4AQAAOILCDwAAwBEUfgAAAI5wdpxLcXGxMZ6QkKDmaK3dZWVlak5kZKQxHhUVpebs2rXLGM/KylJzKisr/dq+iMjJJ59sjMfGxqo5SUlJ6jLAH9oYCRvbKAttXMQll1yi5mzdutUYz8zM9G/HRGTdunXqMu18r6qqUnO074H27durOV27djXG8/Pz1Zyzzz7bGF+2bJmao33foPXZzhvt/Ni5c6eaU1tba4zX1dWpObZxKpqamhq/t6ONUNPGr4iIfPvtt/7tmIO44wcAAOAICj8AAABHUPgBAAA4gsIPAADAERR+AAAAjnC2q3ft2rXGeO/evf1el63DTetkOuWUU9SckpISY9zWPZyYmGiMax1bInqnoS0nOztbXQYESyAPoh8xYoQxPm3aNDVH65A8cOCAmlNUVGSM27qHf/aznxnj//jHP9ScUaNGGeOrVq1Sc7Qu4cGDB6s5559/vjFu61LWvvMC6dSGfwI5xp988om6TPvM2M7B8vJyY9zWbatNxYiJiVFzAvkeOHjwoN85ruGOHwAAgCMo/AAAABxB4QcAAOAICj8AAABHUPgBAAA4gsIPAADAEc6Oc4mNjTXGbaNMtFZ1Wwu79pBp23YiIyONca0d3rZM236geDg7gsX2oHft/LCNQYqIiDDG+/bt69d+tYZ58+YZ49oYJhGRGTNmGOPFxcVqjvY9YDtv77vvPmM8kFEageTAP4EcY9t4orq6OmPcdu3Qrp8+n0/N0a6Thw4dUnO092ob2bJ9+3Z1mcZ2bW2LuOMHAADgCAo/AAAAR1D4AQAAOILCDwAAwBEUfgAAAI5wtqs3LS3N7xytYymQzllbV1J9fb3f29H2TXsAt4jemWV7aLatCxEwCaSzXXsQ/SWXXKLmPP74437t19G0f/9+Y3zEiBFqzsKFC1trd5qgE7ft69q1q7pM64YvLy9Xc6Kjo/3eB+2ap53rInrHsXaNFNGv7bZuX20f2uq5wR0/AAAAR1D4AQAAOILCDwAAwBEUfgAAAI6g8AMAAHAEhR8AAIAjnB3noj0Y2jbKJJAHOWtjVioqKtScYD4AW2vVt+VocRGR+Ph4dRlOLIGMMLB9NjTa6AXbuq666ipj/C9/+Yuaoz2I3rYd7Zy2jZjQ2I5bUVGRMb5p0ya/t2Mb66S9n/Bw/atee6+2cTv+rgvH1uDBg9Vl2ufJ9rPUPuu2ETAa2/mpbcd2Dmgjkj766CO/t9NWcccPAADAERR+AAAAjqDwAwAAcASFHwAAgCMo/AAAABzhbFdvIJ2GgXQ0at102sOnbWydTNr6bN182rKamho1Jzk5WV2G1hVIl10g67N9zm0PR9fk5eUZ47fccoua8+ijjxrjxcXFao6234F049to27Edm5NPPtkYz8nJ8Xv7gfwMbN83wexodK078lgI5PM8cuRIdZnWvW37vtGmYtiuN4Gcn9o1z3aNuvDCC43x+++/X81xDXf8AAAAHEHhBwAA4AgKPwAAAEdQ+AEAADiCwg8AAMARFH4AAACOcHacS1RUlDFuay2PjIz0eztae7ttO9oy25iNqqoqYzwiIkLN0VrlbfumHTcETyBjgzS28RqBjIXQxpKMGTNGzUlMTDTGZ82apeYUFBQY47bPcyAjkjS2URaBHDftZ6qNxbCxjXVKTU01xvft26fmaCOaBg8erOZo3zf//Oc/1RwcO9p5K6J/R9hGs6SkpBjjts+Zdv20nU+BnNOxsbF+57iGO34AAACOoPADAABwBIUfAACAIyj8AAAAHEHhBwAA4Ig23dVr68zz+XxB246t+0nrWArkoem2jkZtO7Z90zp0q6ur1RytozA7O1vNKSoqUpehuUC6Rm2fdX/99Kc/VZcNHz7cGP/LX/6i5mzcuPHH7lKjQLr8bMdGO9cC+RkkJCSoy7KysozxM844Q82ZNm2aMd6uXTs159NPPzXGy8rK1JytW7ca448//riao3Vv0tV7bHXo0MEYt302tXPKdu2orKw0xgP5HrJNMdCWadsX0Tvb+/btq+asX79eXdYWcccPAADAERR+AAAAjqDwAwAAcASFHwAAgCMo/AAAABxB4QcAAOCINj3OJZAH1NsegG5rO/d3O8F+MLXWem/b50BGVmhs4wIY5+IfbSSC7fMcSE7//v2N8YEDB6o52tgD28gW7TNoO9dqa2uN8WB+Zm369eunLjv99NONcdsYnHfeeccY37Rpk5rz8ccfG+O2URZjxowxxq+44go1Z8OGDca47WH3ixYtMsZjYmLUHLS+9u3b+52jnWuRkZFqTk1NjTFuGzmmLdPWJaJ/ng4cOKDmaNfC7t27qzmMcwEAAECbROEHAADgCAo/AAAAR1D4AQAAOILCDwAAwBFtuqvX1pUWFRVljAfSlWTLsT3oWuPz+YzxQLqUbZ1ZWlelrRO4vr7eGNeOJ/xn+zkHM+eTTz4xxs877zw157bbbjPG//nPf6o5paWlfu1XsHXu3Fldpj3U/je/+Y2ao3Xb3nPPPWqOdqwDoX0/iIh88803xrjWiSwism/fPmP8iSeeUHNOPfVUYzwlJUXNQevr06ePMW67RlVUVBjjtm5bbX3R0dFqTjC/1wK5rvbo0cPvnLaKO34AAACOoPADAABwBIUfAACAIyj8AAAAHEHhBwAA4AgKPwAAAEe06XEuXbp0UZdp7ei2USbaGAXbeAVtfbb2ei3H9oB6bZyKre1d2+9AjkEg7fUInkBGJWijfrSHtouIfPrpp8b4WWedpeZoo15sn+fExERjPCQkRM2ZPHmyMX7zzTerOQ899JAxfskll6g5u3fvVpf5y3auae/VNpYiNTXVGN+xY4eak56ebowXFRWpOQUFBcY4Y52OrZycHGPc9v2gLbOda5qDBw+qy7TxYTZVVVXGuO39aPvdrVs3v7ffVnHHDwAAwBEUfgAAAI6g8AMAAHAEhR8AAIAjKPwAAAAc0aZbMePj4/3OsXXbamydeVrnoi2nvr7e733QOplsnZO2ZRqt4zM5OdnvdcFs6NChxvhNN92k5mjddIsWLVJztm3bZoz/3//9n5qzdetWYzwpKUnNGT16tF9xEZGYmBhjfPHixWpOXFycMT527Fg1Z/PmzeoyjXau2TrbtfMmkHPQ9t0xZcoUY7xTp05qjtYlPGrUKDXnm2++McZfeeUVNQetT/ts2D6bWo6tqzeQa1Qg0yo02kQCEX2/U1JS/N5OW8UdPwAAAEdQ+AEAADiCwg8AAMARFH4AAACOoPADAABwBIUfAACAI9r0OBftQe82tgdJB9Iqr7G1sNvGNfjL9n60/fb5fGqONpaib9++as7SpUvVZWhuxYoVxvi+ffvUnC5duhjjlZWVak7Pnj39zklPTzfG8/Ly1Jw9e/YY49HR0WqOdgy0uEhwR4nYRlloD4ivq6vzezuBjIIqKytTc+bPn2+Mb9++Xc0pLS01xqurq9UcbVRWYWGhmoPWd+jQIWPc9nnWPoO264B2XbGdA9qoNNv3jbadQMbJaNcuF3HHDwAAwBEUfgAAAI6g8AMAAHAEhR8AAIAjKPwAAAAc0aa7ehMSEtRlgXTgaV12ts7ZQLqPAulkCqQTOJCHzWvLtG5PBM/69esDWgb/aZ27wc4J5AH1O3bsCGhZMGmd2ji2Avm5aNdC2zVFW2Y7B7Tt2LqHtfOjqqrK7+0kJSWpOa7hjh8AAIAjKPwAAAAcQeEHAADgCAo/AAAAR1D4AQAAOILCDwAAwBFtepxLIO3b2oOkRfTWcls7uraspqbGvx0T+0PTte1ERUWpOdpoFtsx2L9/vzGemJio5gAAWt/OnTuNcduYFW2ZbZyLNlpMGxEmol8/bTmaQEao2XJcwx0/AAAAR1D4AQAAOILCDwAAwBEUfgAAAI6g8AMAAHBEm+7qtXWaal2wtk4mW4esv2zbCaT7KDIy0u+cQN7P3r17/c4BALS+Xbt2+Z2jddXW1dWpOdr1y9Y9HEiONnmitrZWzdG6egOZpNFWcccPAADAERR+AAAAjqDwAwAAcASFHwAAgCMo/AAAABxB4QcAAOCINj3OJSEhQV1WXV1tjNtavrWHTNtoLfFVVVVqjtbCrrWpi+jvx0YbAVNeXq7maO8nOTnZ7+0DAIJH++4OZMyKjTYKzDYCRru22q6rERER/u2YJcd2/XQNd/wAAAAcQeEHAADgCAo/AAAAR1D4AQAAOILCDwAAwBFtuqv3wIED6rIuXboY4xUVFWqO1jG1Y8cONUfrfoqOjlZzkpKSjPFDhw6pOQcPHjTGbZ1M8fHxxri2zyIiGRkZxrjt/QAAWl9ZWZkxbuuc1Tp+bd2+2rXQdr2xXVf8FRYWpi7TOott13bXcMcPAADAERR+AAAAjqDwAwAAcASFHwAAgCMo/AAAABxB4QcAAOCINj3O5cEHH1SXnXfeecb4z372MzVn7NixxnhISIiaoz2Y2vYw6507d/q9HW2cixYXEcnMzDTGa2tr1Zy5c+ca43/961/VHABA69PGrERERKg5CQkJxrg25kVEJDc31xivqqpSc7SRXz6fT83RxrbYRqhp76ewsFDNcQ13/AAAABxB4QcAAOAICj8AAABHUPgBAAA4gsIPAADAESGerXXn+y+0dJS6TusQFhHp2bOnMZ6YmKjmaA+ztuVs3rzZGP/444/VnIKCAmO8pKREzWlrWvjxP6o419AWca4dO4sWLVKXLV261Bjv3bu3mvPee+8Z47Zr4cqVK43xlJQUNUebSlFWVqbm9OvXzxh/6KGH/N7OiepI5xp3/AAAABxB4QcAAOAICj8AAABHUPgBAAA4gsIPAADAERR+AAAAjmjxOBcAAACc2LjjBwAA4AgKPwAAAEdQ+AEAADiCwg8AAMARFH4AAACOoPADAABwBIUfAACAIyj8AAAAHEHhBwAA4AgKPwAAAEdQ+AEAADiCwg8AAMARFH4AAACOoPADAABwBIUfAACAIyj8Wuijjz6S8ePHS4cOHSQyMlIyMjJkwIABMnXq1KO+LwUFBRISEiLPPfec37krVqyQkJAQWbFiRdD3C2gNzz33nISEhDT+iYqKkszMTBk2bJjMnDlTSktLj/UuAieE759Htj9cH9q28GO9AyeCRYsWybhx42To0KEya9YsycrKkuLiYlm3bp28+uqrMnv27GO9i0CbN2/ePMnLy5Pa2lopLS2VDz/8UB566CF55JFHZP78+XLeeecd610Ejmtr1qxp8vf7779fli9fLsuWLWsSP+WUU47mbuEoo/BrgVmzZkmnTp1kyZIlEh7+n0M2ceJEmTVr1jHcM8AdvXr1kv79+zf+fcKECXL77bfLOeecIxdffLFs3bpVMjIyjLmHDh2SmJiYo7WrwHHprLPOavL3tLQ0CQ0NbRb/oRP1/DlR97u18U+9LVBWVibt2rVrUvQdFhr6n0M4f/58GTFihGRlZUl0dLT06NFDpk+fLhUVFU1yrr32WomLi5OvvvpKRo0aJXFxcdK+fXuZOnWqVFdXN3ltUVGRXHrppRIfHy+JiYly2WWXSUlJSbP9WLdunUycOFFycnIkOjpacnJy5PLLL5fCwsIgHQXg+NOhQweZPXu2lJeXy5///GcR+c/59emnn8qIESMkPj5ehg8fLiIiNTU18vvf/17y8vIkMjJS0tLSZPLkybJr164m6122bJkMHTpUUlNTJTo6Wjp06CATJkyQQ4cONb7m6aefllNPPVXi4uIkPj5e8vLy5O677z56bx5oBUOHDpVevXrJqlWrZODAgRITEyM///nPRUTkm2++kSuvvFLS09MlMjJSevToIbNnz5aGhobGfO3XiUy/orR9+3aZOHGiZGdnN/4K1fDhw2X9+vVNcufPny8DBgyQ2NhYiYuLk5EjR8q//vWvJq+xnfdoijt+LTBgwAB55pln5JZbbpErrrhC+vXrJz6fr9nrtm7dKqNGjZLbbrtNYmNj5YsvvpCHHnpI1q5d2+xWem1trYwbN06uu+46mTp1qqxatUruv/9+SUxMlPvuu09ERCorK+W8886ToqIimTlzpnTr1k0WLVokl112WbNtFxQUSPfu3WXixImSkpIixcXF8vTTT8vpp58umzdvlnbt2rXOwQGOsVGjRklYWJisWrWqMVZTUyPjxo2TKVOmyPTp06Wurk4aGhrkoosukg8++EDuvPNOGThwoBQWFsqMGTNk6NChsm7dOomOjpaCggIZPXq0DBo0SObOnStJSUny3XffyeLFi6WmpkZiYmLk1VdflV/+8pfy61//Wh555BEJDQ2Vr776SjZv3nwMjwQQHMXFxXLllVfKnXfeKQ888ICEhobKrl27ZODAgVJTUyP333+/5OTkyFtvvSV33HGHbNu2TebMmeP3dkaNGiX19fUya9Ys6dChg+zevVtWr14t+/bta3zNAw88IPfee69MnjxZ7r33XqmpqZGHH35YBg0aJGvXrm3yz9Km8x4GHo5o9+7d3jnnnOOJiCcins/n8wYOHOjNnDnTKy8vN+Y0NDR4tbW13sqVKz0R8TZs2NC47JprrvFExHvttdea5IwaNcrr3r1749+ffvppT0S8N998s8nrbrjhBk9EvHnz5qn7XFdX5x08eNCLjY31nnjiicb48uXLPRHxli9f7scRAI6defPmeSLi5efnq6/JyMjwevTo4Xnef86vuXPnNnnNK6+84omI9/e//71JPD8/3xMRb86cOZ7ned7rr7/uiYi3fv16dXs333yzl5SUFOhbAo4L11xzjRcbG9skNmTIEE9EvPfff79JfPr06Z6IeB999FGT+C9+8QsvJCTE+/LLLz3P068xX3/9dZPr1u7duz0R8R5//HF1/7755hsvPDzc+/Wvf90kXl5e7mVmZnqXXnppk/diOu/RHP/U2wKpqanywQcfSH5+vjz44INy0UUXyZYtW+Suu+6S3r17y+7du0Xk37etJ02aJJmZmRIWFiY+n0+GDBkiIiKff/55k3WGhITI2LFjm8T69OnT5J9mly9fLvHx8TJu3Lgmr5s0aVKzfTx48KBMmzZNunTpIuHh4RIeHi5xcXFSUVHRbNtAW+N5XrPYhAkTmvz9rbfekqSkJBk7dqzU1dU1/unbt69kZmY2/tNU3759JSIiQm688UZ5/vnnZfv27c3WfcYZZ8i+ffvk8ssvlzfffLPxOwBoC5KTk+Xcc89tElu2bJmccsopcsYZZzSJX3vtteJ5XrN/1TqSlJQUyc3NlYcfflgeffRR+de//tXkn4xFRJYsWSJ1dXVy9dVXNzlno6KiZMiQIcbu4x+e92iOws8P/fv3l2nTpsnf/vY3KSoqkttvv10KCgpk1qxZcvDgQRk0aJB89NFH8vvf/15WrFgh+fn58sYbb4jIv//Z9vtiYmIkKiqqSSwyMlKqqqoa/15WVmb8ZfXMzMxmsUmTJsmTTz4p119/vSxZskTWrl0r+fn5kpaW1mzbQFtSUVEhZWVlkp2d3RiLiYmRhISEJq/buXOn7Nu3TyIiIsTn8zX5U1JS0li85ebmynvvvSfp6enyq1/9SnJzcyU3N1eeeOKJxnVdddVVMnfuXCksLJQJEyZIenq6nHnmmbJ06dKj86aBVpSVldUsVlZWZowfPu/Kysr82kZISIi8//77MnLkSJk1a5b069dP0tLS5JZbbpHy8nIR+fc5KyJy+umnNztn58+f3+x/uEznPZrjd/wC5PP5ZMaMGfLYY4/Jpk2bZNmyZVJUVCQrVqxovMsnIk1+V8Ffqampsnbt2mbxHzZ37N+/X9566y2ZMWOGTJ8+vTFeXV0te/bsCXj7wIlg0aJFUl9fL0OHDm2MhYSENHtdu3btJDU1VRYvXmxcT3x8fON/Dxo0SAYNGiT19fWybt06+eMf/yi33XabZGRkyMSJE0VEZPLkyTJ58mSpqKiQVatWyYwZM2TMmDGyZcsW6dixY3DfJHAUmc6f1NRUKS4ubhYvKioSEWn8PfLDNzR+2KhouivesWNHefbZZ0VEZMuWLfLaa6/Jb37zG6mpqZE//elPjet8/fXXW3ROmfYbzXHHrwVMH3aR//zzbXZ2duMHLjIysslrDncaBmLYsGFSXl4uCxcubBJ/+eWXm/w9JCREPM9rtu1nnnlG6uvrA94+cLz75ptv5I477pDExESZMmWK9bVjxoyRsrIyqa+vl/79+zf7071792Y5YWFhcuaZZ8pTTz0lIiKffPJJs9fExsbKhRdeKPfcc4/U1NTIZ599Fpw3BxxHhg8fLps3b252DrzwwgsSEhIiw4YNExGRnJwcERHZuHFjk9f98Dr2Q926dZN7771Xevfu3biNkSNHSnh4uGzbts14zn5/vBNajjt+LTBy5Eg5+eSTZezYsZKXlycNDQ2yfv16mT17tsTFxcmtt94q2dnZkpycLDfddJPMmDFDfD6fvPTSS7Jhw4aAt3v11VfLY489JldffbX84Q9/kK5du8rbb78tS5YsafK6hIQEGTx4sDz88MPSrl07ycnJkZUrV8qzzz4rSUlJP/LdA8eHTZs2Nf6OT2lpqXzwwQcyb948CQsLkwULFkhaWpo1f+LEifLSSy/JqFGj5NZbb5UzzjhDfD6ffPvtt7J8+XK56KKLZPz48fKnP/1Jli1bJqNHj5YOHTpIVVWVzJ07V0SkcUj0DTfcINHR0XL22WdLVlaWlJSUyMyZMyUxMVFOP/30Vj8WwNF2++23ywsvvCCjR4+W3/3ud9KxY0dZtGiRzJkzR37xi19It27dROTfv4p03nnnycyZMyU5OVk6duwo77//fuOvPR22ceNGufnmm+VnP/uZdO3aVSIiImTZsmWycePGxn+5ysnJkd/97ndyzz33yPbt2+WCCy6Q5ORk2blzp6xdu1ZiY2Plt7/97VE/Fie8Y9xcckKYP3++N2nSJK9r165eXFyc5/P5vA4dOnhXXXWVt3nz5sbXrV692hswYIAXExPjpaWleddff733ySefNOvANXVSeZ7nzZgxw/vhj+Tbb7/1JkyY4MXFxXnx8fHehAkTvNWrVzdb5+HXJScne/Hx8d4FF1zgbdq0yevYsaN3zTXXNL6Orl6caA539R7+ExER4aWnp3tDhgzxHnjgAa+0tLTJ67Xzy/M8r7a21nvkkUe8U0891YuKivLi4uK8vLw8b8qUKd7WrVs9z/O8NWvWeOPHj/c6duzoRUZGeqmpqd6QIUO8hQsXNq7n+eef94YNG+ZlZGR4ERERXnZ2tnfppZd6GzdubL0DAQSZ1tXbs2dP4+sLCwu9SZMmeampqZ7P5/O6d+/uPfzww159fX2T1xUXF3uXXHKJl5KS4iUmJnpXXnmlt27duibXrZ07d3rXXnutl5eX58XGxnpxcXFenz59vMcee8yrq6trsr5//OMf3rBhw7yEhAQvMjLS69ixo3fJJZd47733nvW9wCzE8wztcAAAAGhz+B0/AAAAR1D4AQAAOILCDwAAwBEUfgAAAI6g8AMAAHAEhR8AAIAjKPwAAAAc0eInd/AMPLRFx+MYS86149uNN95ojNueJXrPPfe01u6cMDjXWt8PH9t52B133KHmpKenG+Pao0pFRD788ENj/P/9v/9n2Tuz+++/X102fPhwY7yiokLNKSsrM8bnzZvn346dwI50rnHHDwAAwBEUfgAAAI6g8AMAAHAEhR8AAIAjWtzcAQDH2ogRI9Rld999tzG+ceNGNWfbtm3G+L333qvmbN261RiPiYlRc3r16mWMX3TRRWoO4C+tuSM1NVXNSUxMNMYPHDig5owcOdIY//jjj9Wc6upqY/yss85Sc7p06WKM2xpPsrOz1WX4N+74AQAAOILCDwAAwBEUfgAAAI6g8AMAAHAEhR8AAIAjKPwAAAAcwTgXAK0qPNz8NVNXV6fmJCUlGeN//OMf1ZyEhARjPC8vT83Rnvn52WefqTn79u0zxrWxGCIiPXv2NMZt42neffdddRlgoo0NCgsLU3O++OILY9z2PFxtbIx2rtv2wXZ+avsWGxur5nTq1MkYT0lJUXP27NmjLmuLuOMHAADgCAo/AAAAR1D4AQAAOILCDwAAwBEUfgAAAI6gqxdAq2poaPA7Jy4uzhivr69Xc7RuW9vD5nfs2GGMa52BInp3om3fNOecc466jK5e+Ev73AbSbZuenq7maB2/tvOmqKjIGLedn6mpqcZ4VVWVmqNNCzjllFPUnA8//FBd1hZxxw8AAMARFH4AAACOoPADAABwBIUfAACAIyj8AAAAHEHhBwAA4AjGuQBoVYGMc/nNb35jjNtGpmhjIcrKytSc0FDz//tGR0erOdpoDNtYCs/zjPHevXurOYC/YmNjjfHS0lK/16WNeRHRR6ZocRF9BIxtbMyePXuMcds5ra2va9euag7jXAAAANAmUfgBAAA4gsIPAADAERR+AAAAjqDwAwAAcARdvQCOO5deeqkxvmPHDjVH67a1PZxd6zjWunBFRHw+nzFu6048dOiQMR4ZGanmAP6KiYkxxqurq9UcrXvX1kGfkpLi346J3nGsnbcieiewti6bqKgov3PaKu74AQAAOILCDwAAwBEUfgAAAI6g8AMAAHAEhR8AAIAjKPwAAAAcwTgXAK0qJCTEGLeNTHnqqaeM8UmTJqk55eXlxnhNTY2ao42/2L17t5qjPez9l7/8pZozdepUYzwrK0vNAfyljWCpq6vze11VVVXqssTERGPcNmZFG12kjWwRCWy/tfUxOuk/uOMHAADgCAo/AAAAR1D4AQAAOILCDwAAwBEUfgAAAI6gqxdAq7J172o6d+5sjB86dEjN0boQA9l+Wlqaukx74P1LL72k5tx5553GeEJCgpqjPVTe1m0Jt4WFhRnjto7W8HBzGWDL2b9/vzEeSJd6aWmpukzrEtbOQduy1NRU/3asDeOOHwAAgCMo/AAAABxB4QcAAOAICj8AAABHUPgBAAA4gsIPAADAEYxzaWWBPKBec/7556vLPvvsM2O8qKjI7+0cLdqxsS2z5WgPKMfxqX///n4v27dvn5rj8/mM8dBQ/f9vtYfAZ2dnqzn5+fnqMo12fp599tlqju2zDvhDGw1ko41SEdFHsFx55ZVqzvPPP2+MFxQUqDnaWCXbvmnjaRITE9Uc13DHDwAAwBEUfgAAAI6g8AMAAHAEhR8AAIAjKPwAAAAcQVevHwLp0A0kJy8vzxi/55571JxrrrnGGB83bpyas3z5cmO8vLxczdFoDwcX0bttbccgkK5nnFhuuukmdVltba1fcRH9M2j7bGpdvbaO2o8//lhdptEeHG/rOI6OjjbGKysr/d4+3BYZGaku084B7TMronfO/vOf/1RztAkTWVlZao7Gdk5XVVUZ47Zj4Bru+AEAADiCwg8AAMARFH4AAACOoPADAABwBIUfAACAIyj8AAAAHOHsOBdtXEMgY0lsox8CGUuyZcsWY/y3v/2tmqONbRkxYoSa07dvX2N85syZao42TkM7NjZJSUnqsiFDhhjjJ598sprz17/+1Rjfv3+/X/uFo6Nbt27qMp/PZ4xrYyRE9NEotpEp2vlZWFio5mhjVmw+//xzY/yiiy5Sc0aOHGmMv/LKK35vH26wjWDxl+1789xzzzXG/+d//kfN6dKlizGenp6u5lRUVBjjUVFRao52DdfW5SLu+AEAADiCwg8AAMARFH4AAACOoPADAABwBIUfAACAI350V6+to1UTSKerrdtWW19DQ4PfOdoDq4PNdty0/V6+fLma07FjR2Pc1gXZtWtXY3z16tVqzvvvv2+Mb9++Xc3Jzc01xrUuLxGR888/3xj/5JNP1JwNGzYY4x9++KGag2PHdq4F0kGvLbPlaB26hw4dUnMGDRqkLtNo50dlZaWaYzt3gdZmm9SgXY8TEhLUnOLiYmM8Ozvbvx2zbF9EP2927Njh93baKu74AQAAOILCDwAAwBEUfgAAAI6g8AMAAHAEhR8AAIAjKPwAAAAc0eJ5AYGMbQmE9vDlqqqqo7L90047TV328ccf+70+7bgFMtLG5rnnnjPGX3rpJTXniSeeMMb79Omj5iQnJxvjttb/mJgYY/ydd95Rcz766CNjfOXKlWrOt99+qy7D8SeQEU22ETDx8fHGeE1NjZqjrc+2nZNOOkldpnn99deN8bfeekvNOVrfeWg7Dhw4YIxHRkaqOdpn3fb5S01NNcZTUlLUHG2cS/v27dWcoqIiYzwxMVHN0ZSVlfmd01Zxxw8AAMARFH4AAACOoPADAABwBIUfAACAIyj8AAAAHPGjnwIe7O7UQDrZ0tPTjfEJEyaoORdffLExfu6556o5WveR1nkkoncu2roGg9kJXFtbqy775z//aYyfddZZas7evXuNca3LS0Tk4YcfNsb/+te/qjl0NLZ90dHRfucEcg5okwJEREJDzf/vW1lZqeaUlpYa4+eff76as3TpUmOczzmCaevWrca47bNZUVFhjIeH6+VBWlqaMb5jxw41Z8+ePca47VpoW6bROn61rmIXcccPAADAERR+AAAAjqDwAwAAcASFHwAAgCMo/AAAABxB4QcAAOCIFo9zCWSMgtYObmvRHjhwoDE+Z84cNUcbC1FeXq7mHDx40BjPz89Xc/r372+ML1y4UM2pr69Xlx1r7777rjH+wgsvqDna2JZvvvlGzXnmmWf82zELbdSNSPBHC6F1ZWRkqMu0cSoRERFqjrbM9pmxjTvSaCNgTj31VDVHG+diez81NTX+7Ricp11vbNdcLcc2zkVbZhuZoo160caviIjExsaqyzTaeDdtnIyLuOMHAADgCAo/AAAAR1D4AQAAOILCDwAAwBEUfgAAAI5ocVdvIALpaNW6hWwdc4sXLzbGw8LC/N6OrZPusssuM8ZtXb22jkKN1jVo61rVltlyBg0aZIzbHhzv8/mM8ezsbDVHY/v5BEI7boE86ButLy4uTl2mPTg+kJ+l7XtImwigdRWL6OdUp06d/NsxoXMXwaWdN9XV1X6vKyUlRV2mXT+/+OILNUfrqrVtJyoqSl2m0b4j6Or9D+74AQAAOILCDwAAwBEUfgAAAI6g8AMAAHAEhR8AAIAjKPwAAAAc0arjXGyjRDSrVq0yxr/88ks1R3uQs22USmRkpDGujXcQCWxcQ0NDg985gYzBCYR2fObMmaPmaCNYUlNT/d5+sN/n0Tpu8I82Asj2AHbtu8M23kEb53Pw4EE1R/seiIiI8Hs7HTp0UHOAo0EbZRLIGCTt3LAJZGSKLSeQkV/a6Bpt1I2LuOMHAADgCAo/AAAAR1D4AQAAOILCDwAAwBEUfgAAAI5ocVfvxIkTjfG+ffuqOStXrjTGbd01n376qTG+a9cuNUfrtj106JCao3Ua2h7OrnWNXnjhhWrOO++8Y4zbOhrj4+ON8dzcXDUnPT1dXaYZM2aMMd6rVy81Z8uWLcZ4+/bt1ZzLL7/cGC8vL1dztM4s28PGExISjPENGzaoOWh9WmeerWNP65y15WidwBkZGX7nBNKNH8g5CASTdm0ND9cv9do5Zeug175rA1FWVqYu0/bB1nGsdTDT1fsf3PEDAABwBIUfAACAIyj8AAAAHEHhBwAA4AgKPwAAAEdQ+AEAADiixeNcli5daozv27dPzdEeWm4bezBgwABjXBsjIqKP+Ni7d6+ak5KSoi7TpKWlGeMzZsxQc8aNG2eM29rha2pqjPEDBw6oOdrPwfYA7E8++cQY/+6779Sc/fv3G+ObN29Wc0477TRj3NaSHx0dbYzbxnloYzvmzZun5qD1ZWZmGuMhISF+r8s2zicmJsYYt53rX375pTFu2zdt1Esg3ylAMGmjTLS4iD7qxfb9XFxc7N+OWdjOaW0fbONptLEttmPgGu74AQAAOILCDwAAwBEUfgAAAI6g8AMAAHAEhR8AAIAjWtzVqz1IefHixUHbGRtbl532oHUbn89njNs686qqqvxal4hIRESEMV5UVGTZu2MrKytLXaZ12x46dEjN0bqUbcdN+5nW1taqOdqxtnU2o/VlZ2cb44F09YaG6v+vqn3OtLiI3lFomzygrU/rKg6UdnwC+b6D2+rr69VlWuesrau3tLT0R+/TYTt27FCXderUyRi3dQLbOn7xb9zxAwAAcASFHwAAgCMo/AAAABxB4QcAAOAICj8AAABHUPgBAAA44oTpew72CANtLMjOnTuDup0TUTAfwA3Ex8cb4w0NDWqO9kB126gG7Zy2jbIICwszxm1jY7QxK7bxRMDRYBtzoomKijLGY2Nj1Zz9+/f7vR2NbZ+181MbrSaij2SzvR/bOLK2iDt+AAAAjqDwAwAAcASFHwAAgCMo/AAAABxB4QcAAOCIE6arF8CJKTMz0xi3deprXb22btvo6Gi/cyorK9VlmpqaGmO8sLDQ73XZaN3DwZ5wgLajoqLCGI+MjFRztHPNlhPMyQ/a9m1snfo4Mu74AQAAOILCDwAAwBEUfgAAAI6g8AMAAHAEhR8AAIAjKPwAAAAcwTgXAK0qIyPDGNfGlYiIhIebv5oaGhrUHG19ttEP2voiIiLUHG3MRbBHTNiOD2BiG8Gi2b9/vzFu+zyXlpb6vR1NIONc8ONwxw8AAMARFH4AAACOoPADAABwBIUfAACAIyj8AAAAHEFXL4BWlZaWZozbOnR9Pp8xXlVVpeZoXYg1NTWWvTMLCwtTl2n7HRrK/0fj2KqurjbGbd2+sbGxfm/nq6++8jtHY+seTk1NNcZ37doVtO27iG8qAAAAR1D4AQAAOILCDwAAwBEUfgAAAI6g8AMAAHAEhR8AAIAjGOcCoFVp4yJsD3oPCQkxxm0jYMLDzV9ntnER2tiWysrKoG4nOjra7+1oxwDQpKSkGOO2cS4HDhxord1pEdv3gG18k6aurs4Yt52fruGOHwAAgCMo/AAAABxB4QcAAOAICj8AAABHUPgBAAA4gq5eAK1qzJgxfudoHXhax56I3p2YnJys5nieZ4zbuodramqMcdvD7nNzc43xTZs2qTl09cJfqampxnhUVJSaU1FRYYzbPs8JCQn+7ZiF7ZzWvgds3b7a+7F1NruGO34AAACOoPADAABwBIUfAACAIyj8AAAAHEHhBwAA4AgKPwAAAEcwzgVAq/r5z39ujC9YsEDN0UYvZGZmqjnx8fHGeHi4/jXXrl07Y7y6ulrN0cZFaGNejrQMCJasrCxj/LTTTlNz9uzZY4ynpKQEZZ+OpH379uqy9PR0Y3z//v1qzvnnn2+M//GPf1RziouL1WVtEXf8AAAAHEHhBwAA4AgKPwAAAEdQ+AEAADiCwg8AAMARIZ72lPIfvpAHhqMNauHH/6hy5VzTuu9EREaOHGmM246N9rPctm2bmlNZWWmMd+rUSc0pKSkxxj///HM1Z8WKFeoyV3CuHTsXXHCBuiwvL88Yt3XD/8///I8xfuDAAf92TERycnLUZSNGjDDGw8LC1Jyvv/7aGF+8eLFf+3UiO9K5xh0/AAAAR1D4AQAAOILCDwAAwBEUfgAAAI6g8AMAAHAEhR8AAIAjWjzOBQAAACc27vgBAAA4gsIPAADAERR+AAAAjqDwAwAAcASFHwAAgCMo/AAAABxB4QcAAOAICj8AAABHUPgBAAA44v8Dorf1bUnBn70AAAAASUVORK5CYII="
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 2
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## Creating a Custom Dataset for your files",
   "id": "ffd3136e35e25383"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T12:37:46.865509Z",
     "start_time": "2024-10-11T12:37:46.544544Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from torchvision.io import read_image\n",
    "\n",
    "class CustomImageDataset(Dataset):\n",
    "    def __init__(self, annotations_file, img_dir, transform=None, target_transform=None):\n",
    "        self.img_labels = pd.read_csv(annotations_file)\n",
    "        self.img_dir = img_dir\n",
    "        self.transform = transform\n",
    "        self.target_transform = target_transform\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.img_labels)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        img_path = os.path.join(self.img_dir, self.img_labels.iloc[idx, 0])\n",
    "        image = read_image(img_path)\n",
    "        label = self.img_labels.iloc[idx, 1]\n",
    "        if self.transform:\n",
    "            image = self.transform(image)\n",
    "        if self.target_transform:\n",
    "            label = self.target_transform(label)\n",
    "        return image, label\n",
    "    \n"
   ],
   "id": "99637f51939fa690",
   "outputs": [],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T12:38:33.123332Z",
     "start_time": "2024-10-11T12:38:33.119815Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from torch.utils.data import DataLoader\n",
    "\n",
    "train_dataloader = DataLoader(training_data, batch_size=64, shuffle=True)\n",
    "test_dataloader = DataLoader(test_data, batch_size=64, shuffle=True)"
   ],
   "id": "1743c50d9e5ccedb",
   "outputs": [],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T12:39:35.814048Z",
     "start_time": "2024-10-11T12:39:35.589942Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# Display image and label.\n",
    "train_features, train_labels = next(iter(train_dataloader))\n",
    "print(f\"Feature batch shape: {train_features.size()}\")\n",
    "print(f\"Labels batch shape: {train_labels.size()}\")\n",
    "img = train_features[0].squeeze()\n",
    "label = train_labels[0]\n",
    "plt.imshow(img, cmap=\"gray\")\n",
    "plt.show()\n",
    "print(f\"Label: {label}\")"
   ],
   "id": "855b72b541bc80ef",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature batch shape: torch.Size([64, 1, 28, 28])\n",
      "Labels batch shape: torch.Size([64])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAGdCAYAAAC7EMwUAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAbP0lEQVR4nO3dbWhU6RnG8WsSzRjtGDa4yUxqDGFRWoxIq9YXfIlSg4FK3WypuwutQiv7ooLNLtJUqKEfzNai+MGupUuxSrVrP7hWUNZNq4ldrEWtW8VdrFtjjZiQmq6ZGOPEJE8/yA4dX/ccZ+bOZP4/OGDOnNtzezx65cnM3BNwzjkBAGAgx7oBAED2IoQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABgZoR1A/cbHBzU9evXFQqFFAgErNsBAHjknFN3d7dKSkqUk/P4tc6QC6Hr16+rtLTUug0AwFNqbW3V+PHjH3vMkAuhUChk3QJS6Gtf+5rnmtdee81zTV9fn+caSXrmmWc81/T29nqu8TMty0+Nn94kKRwOe6750Y9+5Lnm+vXrnmuQOb7I/+cpC6G3335bv/jFL9TW1qbJkydr27Ztmjdv3hPr+BHc8Jabm+u5Jj8/Py3nkaTRo0d7rvFzz/oJlMHBQc81fvm5Dk/6sQuyzxf5t5GSu2bfvn1at26dNmzYoLNnz2revHmqrq7W1atXU3E6AECGSkkIbd26VT/4wQ/0wx/+UF/96le1bds2lZaWaseOHak4HQAgQyU9hPr6+nTmzBlVVVUl7K+qqtKJEyceOD4WiykajSZsAIDskPQQunHjhgYGBlRcXJywv7i4WO3t7Q8c39DQoIKCgvjGK+MAIHuk7JnE+5+Qcs499Emquro6dXV1xbfW1tZUtQQAGGKS/uq4cePGKTc394FVT0dHxwOrI0kKBoMKBoPJbgMAkAGSvhLKy8vTtGnT1NjYmLC/sbFRc+bMSfbpAAAZLCXvE6qtrdX3vvc9TZ8+XbNnz9avf/1rXb16Va+++moqTgcAyFApCaHly5ers7NTP/vZz9TW1qaKigodPnxYZWVlqTgdACBDBZyft26nUDQaVUFBgXUbSJFt27Z5rvnud7/rueY///mP5xrJ37v+/fwT8vM8qJ8pEH7H9kycONFzzfe//33PNX/4wx881yBzdHV1aezYsY89hjkbAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMEEIAADOEEADADCEEADBDCAEAzKRkijbwKH4GhPoZRhqNRj3XSP6GhPr5M/X19Xmu8dNbd3e35xpJ6uzs9FwTDod9nQvZjZUQAMAMIQQAMEMIAQDMEEIAADOEEADADCEEADBDCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMU7SRVn4mLY8Y4f029VMjSaNGjfJcE4vFPNcMDg56rvEzRTs/P99zjSTl5eV5rpkwYYKvcyG7sRICAJghhAAAZgghAIAZQggAYIYQAgCYIYQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABghgGmSKucHO/f9wwMDHiu8TvA1M+QUD+cc55r/Fy7u3fveq6RpJ6eHs81xcXFvs6F7MZKCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYIYQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBkGmGLIy8vL81xz584dX+caHBz0XDNhwgTPNX6Gst68edNzTX5+vucayd8g12Aw6OtcyG6shAAAZgghAICZpIdQfX29AoFAwhYOh5N9GgDAMJCS54QmT56sP/3pT/Gv0/VBYQCAzJKSEBoxYgSrHwDAE6XkOaFLly6ppKRE5eXlevHFF3X58uVHHhuLxRSNRhM2AEB2SHoIzZw5U7t379aRI0f0zjvvqL29XXPmzFFnZ+dDj29oaFBBQUF8Ky0tTXZLAIAhKukhVF1drRdeeEFTpkzRN7/5TR06dEiStGvXroceX1dXp66urvjW2tqa7JYAAENUyt+sOmbMGE2ZMkWXLl166OPBYJA3uQFAlkr5+4RisZg++eQTRSKRVJ8KAJBhkh5Cb775ppqbm9XS0qK//e1v+s53vqNoNKoVK1Yk+1QAgAyX9B/HXbt2TS+99JJu3LihZ599VrNmzdLJkydVVlaW7FMBADJc0kPo3XffTfZviWHk9u3bnmsCgYDnGj+DSCV/Az/9DBb1Y+zYsZ5rbty44etcfgas+h0ai+zG7DgAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYIYQAAGYIIQCAGUIIAGCGEAIAmCGEAABmUv6hdsD/6+7u9lwzcuRIzzV37971XCPJ1+de/fznP/dcM2rUKM8169ev91zT3t7uuUaSvvSlL3mu8fN3C7ASAgCYIYQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYYYo20srPVGc/E537+/s910hSMBj0XPPf//7Xc83g4KDnmpwc798z+jmP5O86DAwM+DoXshsrIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYIYQAAGYYYIq0+uc//+m5ZsyYMZ5rRo0a5blGkkaM8P5P4uOPP/Zcc+vWLc81fnobPXq05xrJ3wDYnp4eX+dCdmMlBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMEEIAADOEEADADCEEADBDCAEAzBBCAAAzhBAAwAwDTJFWN27c8FyTk+P9e6Xc3FzPNZIUCAQ81/zjH//wXHPz5k3PNUMdA0zhByshAIAZQggAYMZzCB0/flxLly5VSUmJAoGADhw4kPC4c0719fUqKSlRfn6+KisrdeHChWT1CwAYRjyHUE9Pj6ZOnart27c/9PHNmzdr69at2r59u06dOqVwOKzFixeru7v7qZsFAAwvnl+YUF1drerq6oc+5pzTtm3btGHDBtXU1EiSdu3apeLiYu3du1evvPLK03ULABhWkvqcUEtLi9rb21VVVRXfFwwGtWDBAp04ceKhNbFYTNFoNGEDAGSHpIZQe3u7JKm4uDhhf3Fxcfyx+zU0NKigoCC+lZaWJrMlAMAQlpJXx93/Xgvn3CPff1FXV6eurq741tramoqWAABDUFLfrBoOhyXdWxFFIpH4/o6OjgdWR58LBoMKBoPJbAMAkCGSuhIqLy9XOBxWY2NjfF9fX5+am5s1Z86cZJ4KADAMeF4J3bp1S59++mn865aWFn300UcqLCzUhAkTtG7dOm3atEkTJ07UxIkTtWnTJo0ePVovv/xyUhsHAGQ+zyF0+vRpLVy4MP51bW2tJGnFihX67W9/q/Xr16u3t1evv/66PvvsM82cOVMffPCBQqFQ8roGAAwLnkOosrJSzrlHPh4IBFRfX6/6+vqn6QvDVH9/f1rO43eA6ePu7UdJ1zDSgYEBzzV+hr9K0ogR3p8uvnXrlq9zIbsxOw4AYIYQAgCYIYQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJghhAAAZgghAIAZQggAYCapn6wKPImfSct+JkE/6uPkn8TPpOp0icVinmv8ThP3M0W7u7vb17mQ3VgJAQDMEEIAADOEEADADCEEADBDCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMMMAU6RVf3+/55qrV696rvE7uNPPkNB0uXPnjucav4NcnXOea/wMpwVYCQEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMEEIAADOEEADADCEEADDDAFOklZ8hl4ODg55rcnL8fX919+5dX3XpcPPmTc81fq+DnwGmnZ2dvs6F7MZKCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYIYQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBkGmCKtRo4cmZbz5Obm+qqLxWJJ7iR5/AxX9TvA9Nq1a55rhvLwVwxdrIQAAGYIIQCAGc8hdPz4cS1dulQlJSUKBAI6cOBAwuMrV65UIBBI2GbNmpWsfgEAw4jnEOrp6dHUqVO1ffv2Rx6zZMkStbW1xbfDhw8/VZMAgOHJ8wsTqqurVV1d/dhjgsGgwuGw76YAANkhJc8JNTU1qaioSJMmTdKqVavU0dHxyGNjsZii0WjCBgDIDkkPoerqau3Zs0dHjx7Vli1bdOrUKS1atOiRL31taGhQQUFBfCstLU12SwCAISrp7xNavnx5/NcVFRWaPn26ysrKdOjQIdXU1DxwfF1dnWpra+NfR6NRgggAskTK36waiURUVlamS5cuPfTxYDCoYDCY6jYAAENQyt8n1NnZqdbWVkUikVSfCgCQYTyvhG7duqVPP/00/nVLS4s++ugjFRYWqrCwUPX19XrhhRcUiUR05coV/eQnP9G4ceP0/PPPJ7VxAEDm8xxCp0+f1sKFC+Nff/58zooVK7Rjxw6dP39eu3fv1s2bNxWJRLRw4ULt27dPoVAoeV0DAIYFzyFUWVkp59wjHz9y5MhTNYThbcQI709D9vf3e67Jy8vzXCMN7QGmfoayPu7f6uP4GXyaruG0GF6YHQcAMEMIAQDMEEIAADOEEADADCEEADBDCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMJPyT1YF/p+fSdB+agYHBz3XSNKdO3d81aVDX1+f5xq/n1rsZ/q234ndyG6shAAAZgghAIAZQggAYIYQAgCYIYQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJhhgCnSKhAIeK7xM8DU7zBNP0NC08XPcNVRo0b5OhfDSJEurIQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYYYAphrycHO/fKw3HAZx37971XOPn2knS4OCgrzrAK1ZCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMEEIAADOEEADADCEEADBDCAEAzDDAFENeIBDwXON3AGcwGPRVlw5+hrKOGOHvn/jAwIDnmlgs5utcyG6shAAAZgghAIAZTyHU0NCgGTNmKBQKqaioSMuWLdPFixcTjnHOqb6+XiUlJcrPz1dlZaUuXLiQ1KYBAMODpxBqbm7W6tWrdfLkSTU2Nqq/v19VVVXq6emJH7N582Zt3bpV27dv16lTpxQOh7V48WJ1d3cnvXkAQGbz9Kzl+++/n/D1zp07VVRUpDNnzmj+/Plyzmnbtm3asGGDampqJEm7du1ScXGx9u7dq1deeSV5nQMAMt5TPSfU1dUlSSosLJQktbS0qL29XVVVVfFjgsGgFixYoBMnTjz094jFYopGowkbACA7+A4h55xqa2s1d+5cVVRUSJLa29slScXFxQnHFhcXxx+7X0NDgwoKCuJbaWmp35YAABnGdwitWbNG586d0+9///sHHrv/fR3OuUe+16Ourk5dXV3xrbW11W9LAIAM4+udbGvXrtXBgwd1/PhxjR8/Pr4/HA5LurciikQi8f0dHR0PrI4+FwwGh/QbBAEAqeNpJeSc05o1a7R//34dPXpU5eXlCY+Xl5crHA6rsbExvq+vr0/Nzc2aM2dOcjoGAAwbnlZCq1ev1t69e/XHP/5RoVAo/jxPQUGB8vPzFQgEtG7dOm3atEkTJ07UxIkTtWnTJo0ePVovv/xySv4AAIDM5SmEduzYIUmqrKxM2L9z506tXLlSkrR+/Xr19vbq9ddf12effaaZM2fqgw8+UCgUSkrDAIDhw1MIfZEBioFAQPX19aqvr/fbE/DU/Aw9lfwNCR3KhtufB8MPs+MAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYIYQAAGZ8fbIqkE7pnAQ9lKdODw4Oeq5J5zRxP/0BrIQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYYYAp0qq/v99zjZ/BmH4HkQ4MDPiqS4ecHO/fM6ZzICsDTOEHKyEAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYIYQAAGYIIQCAGUIIAGCGEAIAmCGEAABmGGCKtOrt7fVc42cwZl5enucaSbp7966vunR45pln0nauQCDguaa7uzsFnWC4YyUEADBDCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMEEIAADOEEADADANMkVZ+BmPm5Hj/Xsk557lGknp6enzVpYOf4ap+r4OfobH9/f2+zoXsxkoIAGCGEAIAmPEUQg0NDZoxY4ZCoZCKioq0bNkyXbx4MeGYlStXKhAIJGyzZs1KatMAgOHBUwg1Nzdr9erVOnnypBobG9Xf36+qqqoHfo6+ZMkStbW1xbfDhw8ntWkAwPDg6YUJ77//fsLXO3fuVFFRkc6cOaP58+fH9weDQYXD4eR0CAAYtp7qOaGuri5JUmFhYcL+pqYmFRUVadKkSVq1apU6Ojoe+XvEYjFFo9GEDQCQHXyHkHNOtbW1mjt3rioqKuL7q6urtWfPHh09elRbtmzRqVOntGjRIsVisYf+Pg0NDSooKIhvpaWlflsCAGQY3+8TWrNmjc6dO6cPP/wwYf/y5cvjv66oqND06dNVVlamQ4cOqaam5oHfp66uTrW1tfGvo9EoQQQAWcJXCK1du1YHDx7U8ePHNX78+MceG4lEVFZWpkuXLj308WAwqGAw6KcNAECG8xRCzjmtXbtW7733npqamlReXv7Ems7OTrW2tioSifhuEgAwPHl6Tmj16tX63e9+p7179yoUCqm9vV3t7e3q7e2VJN26dUtvvvmm/vrXv+rKlStqamrS0qVLNW7cOD3//PMp+QMAADKXp5XQjh07JEmVlZUJ+3fu3KmVK1cqNzdX58+f1+7du3Xz5k1FIhEtXLhQ+/btUygUSlrTAIDhwfOP4x4nPz9fR44ceaqGAADZgynaSKuxY8d6rhk9erTnmtzcXM81kvTcc8/5qksHP68avXPnjq9z+Zm+zU874AcDTAEAZgghAIAZQggAYIYQAgCYIYQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJhhgCnS6u9//7vnmj//+c+ea8rKyjzXSNLBgwd91aXDT3/6U881q1at8nWuf/3rX55rTp8+7etcyG6shAAAZgghAIAZQggAYIYQAgCYIYQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABgZsjNjnPOWbeAIaa3t9dzTU9Pj69zxWIxX3Xp4Ke3W7du+TrX7du3fdUB/++L/H8ecEPsf/1r166ptLTUug0AwFNqbW3V+PHjH3vMkAuhwcFBXb9+XaFQSIFAIOGxaDSq0tJStba2auzYsUYd2uM63MN1uIfrcA/X4Z6hcB2cc+ru7lZJSYlych7/rM+Q+3FcTk7OE5Nz7NixWX2TfY7rcA/X4R6uwz1ch3usr0NBQcEXOo4XJgAAzBBCAAAzGRVCwWBQGzduVDAYtG7FFNfhHq7DPVyHe7gO92TadRhyL0wAAGSPjFoJAQCGF0IIAGCGEAIAmCGEAABmMiqE3n77bZWXl2vUqFGaNm2a/vKXv1i3lFb19fUKBAIJWzgctm4r5Y4fP66lS5eqpKREgUBABw4cSHjcOaf6+nqVlJQoPz9flZWVunDhgk2zKfSk67By5coH7o9Zs2bZNJsiDQ0NmjFjhkKhkIqKirRs2TJdvHgx4ZhsuB++yHXIlPshY0Jo3759WrdunTZs2KCzZ89q3rx5qq6u1tWrV61bS6vJkyerra0tvp0/f966pZTr6enR1KlTtX379oc+vnnzZm3dulXbt2/XqVOnFA6HtXjxYnV3d6e509R60nWQpCVLliTcH4cPH05jh6nX3Nys1atX6+TJk2psbFR/f7+qqqoSBtZmw/3wRa6DlCH3g8sQ3/jGN9yrr76asO8rX/mK+/GPf2zUUfpt3LjRTZ061boNU5Lce++9F/96cHDQhcNh99Zbb8X33blzxxUUFLhf/epXBh2mx/3XwTnnVqxY4b797W+b9GOlo6PDSXLNzc3Ouey9H+6/Ds5lzv2QESuhvr4+nTlzRlVVVQn7q6qqdOLECaOubFy6dEklJSUqLy/Xiy++qMuXL1u3ZKqlpUXt7e0J90YwGNSCBQuy7t6QpKamJhUVFWnSpElatWqVOjo6rFtKqa6uLklSYWGhpOy9H+6/Dp/LhPshI0Loxo0bGhgYUHFxccL+4uJitbe3G3WVfjNnztTu3bt15MgRvfPOO2pvb9ecOXPU2dlp3ZqZz//+s/3ekKTq6mrt2bNHR48e1ZYtW3Tq1CktWrRoSH9G0tNwzqm2tlZz585VRUWFpOy8Hx52HaTMuR+G3BTtx7n/ox2ccw/sG86qq6vjv54yZYpmz56t5557Trt27VJtba1hZ/ay/d6QpOXLl8d/XVFRoenTp6usrEyHDh1STU2NYWepsWbNGp07d04ffvjhA49l0/3wqOuQKfdDRqyExo0bp9zc3Ae+k+no6HjgO55sMmbMGE2ZMkWXLl2ybsXM568O5N54UCQSUVlZ2bC8P9auXauDBw/q2LFjCR/9km33w6Ouw8MM1fshI0IoLy9P06ZNU2NjY8L+xsZGzZkzx6gre7FYTJ988okikYh1K2bKy8sVDocT7o2+vj41Nzdn9b0hSZ2dnWptbR1W94dzTmvWrNH+/ft19OhRlZeXJzyeLffDk67DwwzZ+8HwRRGevPvuu27kyJHuN7/5jfv444/dunXr3JgxY9yVK1esW0ubN954wzU1NbnLly+7kydPum9961suFAoN+2vQ3d3tzp49686ePeskua1bt7qzZ8+6f//7384559566y1XUFDg9u/f786fP+9eeuklF4lEXDQaNe48uR53Hbq7u90bb7zhTpw44VpaWtyxY8fc7Nmz3Ze//OVhdR1ee+01V1BQ4JqamlxbW1t8u337dvyYbLgfnnQdMul+yJgQcs65X/7yl66srMzl5eW5r3/96wkvR8wGy5cvd5FIxI0cOdKVlJS4mpoad+HCBeu2Uu7YsWNO0gPbihUrnHP3Xpa7ceNGFw6HXTAYdPPnz3fnz5+3bToFHncdbt++7aqqqtyzzz7rRo4c6SZMmOBWrFjhrl69at12Uj3szy/J7dy5M35MNtwPT7oOmXQ/8FEOAAAzGfGcEABgeCKEAABmCCEAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYIYQAAGYIIQCAGUIIAGDmf6MtZLI7zO8bAAAAAElFTkSuQmCC"
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label: 1\n"
     ]
    }
   ],
   "execution_count": 5
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Transforms",
   "id": "772bbf511d88c53d"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T12:42:40.960932Z",
     "start_time": "2024-10-11T12:42:40.926595Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import torch\n",
    "from torchvision import datasets\n",
    "from torchvision.transforms import ToTensor, Lambda\n",
    "\n",
    "ds = datasets.FashionMNIST(\n",
    "    root=\"data\",\n",
    "    train=True,\n",
    "    download=True,\n",
    "    transform=ToTensor(),\n",
    "    target_transform=Lambda(lambda y: torch.zeros(10, dtype=torch.float).scatter_(0, torch.tensor(y), value=1))\n",
    ")"
   ],
   "id": "9334ab239d354b7b",
   "outputs": [],
   "execution_count": 7
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Build the Neural Network",
   "id": "b6709b911812a61d"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T12:45:59.382918Z",
     "start_time": "2024-10-11T12:45:59.379906Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import os\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets, transforms"
   ],
   "id": "5408b39c9e715e62",
   "outputs": [],
   "execution_count": 8
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T12:46:24.423565Z",
     "start_time": "2024-10-11T12:46:24.416580Z"
    }
   },
   "cell_type": "code",
   "source": [
    "device = (\n",
    "    \"cuda\"\n",
    "    if torch.cuda.is_available()\n",
    "    else \"mps\"\n",
    "    if torch.backends.mps.is_available()\n",
    "    else \"cpu\"\n",
    ")\n",
    "print(f\"Using {device} device\")"
   ],
   "id": "75f87585647c8b51",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cpu device\n"
     ]
    }
   ],
   "execution_count": 9
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T12:48:39.550092Z",
     "start_time": "2024-10-11T12:48:39.546691Z"
    }
   },
   "cell_type": "code",
   "source": [
    "class NeuralNetwork(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.flatten = nn.Flatten()\n",
    "        self.linear_relu_stack = nn.Sequential(\n",
    "            nn.Linear(28*28, 512),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(512, 512),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(512, 10),\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.flatten(x)\n",
    "        logits = self.linear_relu_stack(x)\n",
    "        return logits"
   ],
   "id": "2026c61de77b3934",
   "outputs": [],
   "execution_count": 10
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T12:48:48.843979Z",
     "start_time": "2024-10-11T12:48:48.777009Z"
    }
   },
   "cell_type": "code",
   "source": [
    "model = NeuralNetwork().to(device)\n",
    "print(model)"
   ],
   "id": "58527e1a88c8d4a5",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NeuralNetwork(\n",
      "  (flatten): Flatten(start_dim=1, end_dim=-1)\n",
      "  (linear_relu_stack): Sequential(\n",
      "    (0): Linear(in_features=784, out_features=512, bias=True)\n",
      "    (1): ReLU()\n",
      "    (2): Linear(in_features=512, out_features=512, bias=True)\n",
      "    (3): ReLU()\n",
      "    (4): Linear(in_features=512, out_features=10, bias=True)\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "execution_count": 11
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T12:49:13.911670Z",
     "start_time": "2024-10-11T12:49:13.652147Z"
    }
   },
   "cell_type": "code",
   "source": [
    "X = torch.rand(1, 28, 28, device=device)\n",
    "logits = model(X)\n",
    "pred_probab = nn.Softmax(dim=1)(logits)\n",
    "y_pred = pred_probab.argmax(1)\n",
    "print(f\"Predicted class: {y_pred}\")"
   ],
   "id": "81c088f57002a61e",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted class: tensor([0])\n"
     ]
    }
   ],
   "execution_count": 12
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T12:49:33.934255Z",
     "start_time": "2024-10-11T12:49:33.930763Z"
    }
   },
   "cell_type": "code",
   "source": [
    "input_image = torch.rand(3,28,28)\n",
    "print(input_image.size())"
   ],
   "id": "1d461cd432a292eb",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([3, 28, 28])\n"
     ]
    }
   ],
   "execution_count": 13
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T12:50:00.680836Z",
     "start_time": "2024-10-11T12:50:00.676944Z"
    }
   },
   "cell_type": "code",
   "source": [
    "flatten = nn.Flatten()\n",
    "flat_image = flatten(input_image)\n",
    "print(flat_image.size())"
   ],
   "id": "d44fd4b5231dc80e",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([3, 784])\n"
     ]
    }
   ],
   "execution_count": 14
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T12:50:26.317146Z",
     "start_time": "2024-10-11T12:50:26.293489Z"
    }
   },
   "cell_type": "code",
   "source": [
    "layer1 = nn.Linear(in_features=28*28, out_features=20)\n",
    "hidden1 = layer1(flat_image)\n",
    "print(hidden1.size())"
   ],
   "id": "1b4f5718783862f7",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([3, 20])\n"
     ]
    }
   ],
   "execution_count": 15
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T12:50:54.406534Z",
     "start_time": "2024-10-11T12:50:54.401647Z"
    }
   },
   "cell_type": "code",
   "source": [
    "print(f\"Before ReLU: {hidden1}\\n\\n\")\n",
    "hidden1 = nn.ReLU()(hidden1)\n",
    "print(f\"After ReLU: {hidden1}\")"
   ],
   "id": "8a850d72e26f527",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before ReLU: tensor([[ 3.2664e-01,  1.9431e-01, -6.0840e-01, -6.5665e-01, -2.1788e-01,\n",
      "          5.8597e-03,  1.9199e-01, -4.1443e-01, -1.2126e-01,  2.0217e-01,\n",
      "          3.3820e-01, -6.8538e-02, -1.6411e-01,  2.0257e-01,  3.5159e-04,\n",
      "         -3.0114e-02,  2.4934e-01,  3.0360e-01, -1.2485e-01,  5.3446e-01],\n",
      "        [ 4.7125e-01,  3.8580e-01, -4.9145e-01, -7.6401e-01, -1.0710e-01,\n",
      "         -2.1989e-02,  2.7386e-01, -3.7297e-01,  4.7725e-02,  1.7617e-01,\n",
      "          3.6924e-01, -1.0016e-01,  2.2892e-01, -7.2815e-02, -7.4952e-02,\n",
      "          2.6428e-01, -6.6616e-02,  1.3217e-02, -4.9674e-02,  6.4690e-01],\n",
      "        [ 2.9796e-01,  1.8526e-01, -4.9002e-01, -4.7326e-01, -3.0415e-01,\n",
      "         -1.4856e-01,  4.7944e-01, -3.8955e-01, -6.0188e-03,  3.2289e-01,\n",
      "          2.2587e-01,  6.4830e-02, -3.8391e-02,  3.9275e-01,  3.3562e-01,\n",
      "          1.3352e-01,  1.1741e-01,  1.4454e-01,  1.5661e-01,  5.3752e-01]],\n",
      "       grad_fn=<AddmmBackward0>)\n",
      "\n",
      "\n",
      "After ReLU: tensor([[3.2664e-01, 1.9431e-01, 0.0000e+00, 0.0000e+00, 0.0000e+00, 5.8597e-03,\n",
      "         1.9199e-01, 0.0000e+00, 0.0000e+00, 2.0217e-01, 3.3820e-01, 0.0000e+00,\n",
      "         0.0000e+00, 2.0257e-01, 3.5159e-04, 0.0000e+00, 2.4934e-01, 3.0360e-01,\n",
      "         0.0000e+00, 5.3446e-01],\n",
      "        [4.7125e-01, 3.8580e-01, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,\n",
      "         2.7386e-01, 0.0000e+00, 4.7725e-02, 1.7617e-01, 3.6924e-01, 0.0000e+00,\n",
      "         2.2892e-01, 0.0000e+00, 0.0000e+00, 2.6428e-01, 0.0000e+00, 1.3217e-02,\n",
      "         0.0000e+00, 6.4690e-01],\n",
      "        [2.9796e-01, 1.8526e-01, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,\n",
      "         4.7944e-01, 0.0000e+00, 0.0000e+00, 3.2289e-01, 2.2587e-01, 6.4830e-02,\n",
      "         0.0000e+00, 3.9275e-01, 3.3562e-01, 1.3352e-01, 1.1741e-01, 1.4454e-01,\n",
      "         1.5661e-01, 5.3752e-01]], grad_fn=<ReluBackward0>)\n"
     ]
    }
   ],
   "execution_count": 16
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T12:52:00.798003Z",
     "start_time": "2024-10-11T12:52:00.793387Z"
    }
   },
   "cell_type": "code",
   "source": [
    "seq_modules = nn.Sequential(\n",
    "    flatten,\n",
    "    layer1,\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(20, 10)\n",
    ")\n",
    "input_image = torch.rand(3,28,28)\n",
    "logits = seq_modules(input_image)"
   ],
   "id": "5d5c737c72a8904f",
   "outputs": [],
   "execution_count": 17
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T12:52:43.438532Z",
     "start_time": "2024-10-11T12:52:43.434638Z"
    }
   },
   "cell_type": "code",
   "source": [
    "softmax = nn.Softmax(dim=1)\n",
    "pred_probab = softmax(logits)"
   ],
   "id": "442f38de6882ea6d",
   "outputs": [],
   "execution_count": 18
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T12:53:20.784199Z",
     "start_time": "2024-10-11T12:53:20.766313Z"
    }
   },
   "cell_type": "code",
   "source": [
    "print(f\"Model structure: {model}\\n\\n\")\n",
    "\n",
    "for name, param in model.named_parameters():\n",
    "    print(f\"Layer: {name} | Size: {param.size()} | Values : {param[:2]} \\n\")"
   ],
   "id": "9aad24545dcafb50",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model structure: NeuralNetwork(\n",
      "  (flatten): Flatten(start_dim=1, end_dim=-1)\n",
      "  (linear_relu_stack): Sequential(\n",
      "    (0): Linear(in_features=784, out_features=512, bias=True)\n",
      "    (1): ReLU()\n",
      "    (2): Linear(in_features=512, out_features=512, bias=True)\n",
      "    (3): ReLU()\n",
      "    (4): Linear(in_features=512, out_features=10, bias=True)\n",
      "  )\n",
      ")\n",
      "\n",
      "\n",
      "Layer: linear_relu_stack.0.weight | Size: torch.Size([512, 784]) | Values : tensor([[ 0.0174,  0.0315,  0.0064,  ...,  0.0145,  0.0112,  0.0011],\n",
      "        [-0.0030,  0.0191,  0.0284,  ...,  0.0136,  0.0318,  0.0043]],\n",
      "       grad_fn=<SliceBackward0>) \n",
      "\n",
      "Layer: linear_relu_stack.0.bias | Size: torch.Size([512]) | Values : tensor([-0.0022,  0.0188], grad_fn=<SliceBackward0>) \n",
      "\n",
      "Layer: linear_relu_stack.2.weight | Size: torch.Size([512, 512]) | Values : tensor([[ 0.0248, -0.0293,  0.0103,  ...,  0.0305,  0.0190, -0.0020],\n",
      "        [ 0.0337,  0.0134,  0.0386,  ...,  0.0072, -0.0176,  0.0235]],\n",
      "       grad_fn=<SliceBackward0>) \n",
      "\n",
      "Layer: linear_relu_stack.2.bias | Size: torch.Size([512]) | Values : tensor([-0.0203, -0.0390], grad_fn=<SliceBackward0>) \n",
      "\n",
      "Layer: linear_relu_stack.4.weight | Size: torch.Size([10, 512]) | Values : tensor([[ 0.0108,  0.0419,  0.0364,  ...,  0.0002, -0.0328,  0.0195],\n",
      "        [ 0.0080, -0.0407,  0.0043,  ...,  0.0118, -0.0167,  0.0130]],\n",
      "       grad_fn=<SliceBackward0>) \n",
      "\n",
      "Layer: linear_relu_stack.4.bias | Size: torch.Size([10]) | Values : tensor([0.0331, 0.0228], grad_fn=<SliceBackward0>) \n",
      "\n"
     ]
    }
   ],
   "execution_count": 19
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Automatic Differentiation with torch.autograd",
   "id": "c70f5a676a9dd5d7"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T12:55:43.722211Z",
     "start_time": "2024-10-11T12:55:43.598462Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import torch\n",
    "\n",
    "x = torch.ones(5)  # input tensor\n",
    "y = torch.zeros(3)  # expected output\n",
    "w = torch.randn(5, 3, requires_grad=True)\n",
    "b = torch.randn(3, requires_grad=True)\n",
    "z = torch.matmul(x, w)+b\n",
    "loss = torch.nn.functional.binary_cross_entropy_with_logits(z, y)"
   ],
   "id": "21e598bb7866c422",
   "outputs": [],
   "execution_count": 20
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T12:56:39.693353Z",
     "start_time": "2024-10-11T12:56:39.688351Z"
    }
   },
   "cell_type": "code",
   "source": [
    "print(f\"Gradient function for z = {z.grad_fn}\")\n",
    "print(f\"Gradient function for loss = {loss.grad_fn}\")"
   ],
   "id": "120dcd0b87f2fdd5",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient function for z = <AddBackward0 object at 0x000001FA946A7D00>\n",
      "Gradient function for loss = <BinaryCrossEntropyWithLogitsBackward0 object at 0x000001FA92FC7CD0>\n"
     ]
    }
   ],
   "execution_count": 21
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T12:57:12.147598Z",
     "start_time": "2024-10-11T12:57:11.985852Z"
    }
   },
   "cell_type": "code",
   "source": [
    "loss.backward()\n",
    "print(w.grad)\n",
    "print(b.grad)"
   ],
   "id": "f0996fd58a24de92",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.0167, 0.0262, 0.3073],\n",
      "        [0.0167, 0.0262, 0.3073],\n",
      "        [0.0167, 0.0262, 0.3073],\n",
      "        [0.0167, 0.0262, 0.3073],\n",
      "        [0.0167, 0.0262, 0.3073]])\n",
      "tensor([0.0167, 0.0262, 0.3073])\n"
     ]
    }
   ],
   "execution_count": 22
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "> - We can only obtain the grad properties for the leaf nodes of the computational graph, which have requires_grad property set to True. For all other nodes in our graph, gradients will not be available.\n",
    "> - We can only perform gradient calculations using backward once on a given graph, for performance reasons. If we need to do several backward calls on the same graph, we need to pass retain_graph=True to the backward call."
   ],
   "id": "c0bb21612b28113"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T12:59:46.638235Z",
     "start_time": "2024-10-11T12:59:46.633804Z"
    }
   },
   "cell_type": "code",
   "source": [
    "z = torch.matmul(x, w)+b\n",
    "print(z.requires_grad)\n",
    "\n",
    "with torch.no_grad():\n",
    "    z = torch.matmul(x, w)+b\n",
    "print(z.requires_grad)"
   ],
   "id": "ebeb76984c2dd3e8",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "False\n"
     ]
    }
   ],
   "execution_count": 23
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T13:00:05.328590Z",
     "start_time": "2024-10-11T13:00:05.324006Z"
    }
   },
   "cell_type": "code",
   "source": [
    "z = torch.matmul(x, w)+b\n",
    "z_det = z.detach()\n",
    "print(z_det.requires_grad)"
   ],
   "id": "30b6d770cf773727",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n"
     ]
    }
   ],
   "execution_count": 24
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T13:03:25.697515Z",
     "start_time": "2024-10-11T13:03:25.639608Z"
    }
   },
   "cell_type": "code",
   "source": [
    "inp = torch.eye(4, 5, requires_grad=True)\n",
    "out = (inp+1).pow(2).t()\n",
    "out.backward(torch.ones_like(out), retain_graph=True)\n",
    "print(f\"First call\\n{inp.grad}\")\n",
    "out.backward(torch.ones_like(out), retain_graph=True)\n",
    "print(f\"\\nSecond call\\n{inp.grad}\")\n",
    "inp.grad.zero_()\n",
    "out.backward(torch.ones_like(out), retain_graph=True)\n",
    "print(f\"\\nCall after zeroing gradients\\n{inp.grad}\")"
   ],
   "id": "bf3b4a6340e751ad",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First call\n",
      "tensor([[4., 2., 2., 2., 2.],\n",
      "        [2., 4., 2., 2., 2.],\n",
      "        [2., 2., 4., 2., 2.],\n",
      "        [2., 2., 2., 4., 2.]])\n",
      "\n",
      "Second call\n",
      "tensor([[8., 4., 4., 4., 4.],\n",
      "        [4., 8., 4., 4., 4.],\n",
      "        [4., 4., 8., 4., 4.],\n",
      "        [4., 4., 4., 8., 4.]])\n",
      "\n",
      "Call after zeroing gradients\n",
      "tensor([[4., 2., 2., 2., 2.],\n",
      "        [2., 4., 2., 2., 2.],\n",
      "        [2., 2., 4., 2., 2.],\n",
      "        [2., 2., 2., 4., 2.]])\n"
     ]
    }
   ],
   "execution_count": 25
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Optimizing Model Parameters",
   "id": "298d1ae144ff5a45"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T13:06:26.117238Z",
     "start_time": "2024-10-11T13:06:26.073459Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets\n",
    "from torchvision.transforms import ToTensor\n",
    "\n",
    "training_data = datasets.FashionMNIST(\n",
    "    root=\"data\",\n",
    "    train=True,\n",
    "    download=True,\n",
    "    transform=ToTensor()\n",
    ")\n",
    "\n",
    "test_data = datasets.FashionMNIST(\n",
    "    root=\"data\",\n",
    "    train=False,\n",
    "    download=True,\n",
    "    transform=ToTensor()\n",
    ")\n",
    "\n",
    "train_dataloader = DataLoader(training_data, batch_size=64)\n",
    "test_dataloader = DataLoader(test_data, batch_size=64)\n",
    "\n",
    "class NeuralNetwork(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.flatten = nn.Flatten()\n",
    "        self.linear_relu_stack = nn.Sequential(\n",
    "            nn.Linear(28*28, 512),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(512, 512),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(512, 10),\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.flatten(x)\n",
    "        logits = self.linear_relu_stack(x)\n",
    "        return logits\n",
    "\n",
    "model = NeuralNetwork()"
   ],
   "id": "19ead4630b61d797",
   "outputs": [],
   "execution_count": 26
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T13:07:07.388160Z",
     "start_time": "2024-10-11T13:07:07.385011Z"
    }
   },
   "cell_type": "code",
   "source": [
    "learning_rate = 1e-3\n",
    "batch_size = 64\n",
    "epochs = 5"
   ],
   "id": "e22bad0a90c7f766",
   "outputs": [],
   "execution_count": 27
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T13:08:54.907593Z",
     "start_time": "2024-10-11T13:08:54.903418Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# Initialize the loss function\n",
    "loss_fn = nn.CrossEntropyLoss()"
   ],
   "id": "13f3f0d051808521",
   "outputs": [],
   "execution_count": 28
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T13:09:39.154074Z",
     "start_time": "2024-10-11T13:09:39.150160Z"
    }
   },
   "cell_type": "code",
   "source": "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)",
   "id": "1d066a590f9ca6c8",
   "outputs": [],
   "execution_count": 29
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T13:10:55.967858Z",
     "start_time": "2024-10-11T13:10:55.962605Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def train_loop(dataloader, model, loss_fn, optimizer):\n",
    "    size = len(dataloader.dataset)\n",
    "    # Set the model to training mode - important for batch normalization and dropout layers\n",
    "    # Unnecessary in this situation but added for best practices\n",
    "    model.train()\n",
    "    for batch, (X, y) in enumerate(dataloader):\n",
    "        # Compute prediction and loss\n",
    "        pred = model(X)\n",
    "        loss = loss_fn(pred, y)\n",
    "\n",
    "        # Backpropagation\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        if batch % 100 == 0:\n",
    "            loss, current = loss.item(), batch * batch_size + len(X)\n",
    "            print(f\"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]\")\n",
    "\n",
    "\n",
    "def test_loop(dataloader, model, loss_fn):\n",
    "    # Set the model to evaluation mode - important for batch normalization and dropout layers\n",
    "    # Unnecessary in this situation but added for best practices\n",
    "    model.eval()\n",
    "    size = len(dataloader.dataset)\n",
    "    num_batches = len(dataloader)\n",
    "    test_loss, correct = 0, 0\n",
    "\n",
    "    # Evaluating the model with torch.no_grad() ensures that no gradients are computed during test mode\n",
    "    # also serves to reduce unnecessary gradient computations and memory usage for tensors with requires_grad=True\n",
    "    with torch.no_grad():\n",
    "        for X, y in dataloader:\n",
    "            pred = model(X)\n",
    "            test_loss += loss_fn(pred, y).item()\n",
    "            correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
    "\n",
    "    test_loss /= num_batches\n",
    "    correct /= size\n",
    "    print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")"
   ],
   "id": "ea564d63957f7c73",
   "outputs": [],
   "execution_count": 30
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T13:12:55.522284Z",
     "start_time": "2024-10-11T13:11:12.193292Z"
    }
   },
   "cell_type": "code",
   "source": [
    "loss_fn = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n",
    "\n",
    "epochs = 10\n",
    "for t in range(epochs):\n",
    "    print(f\"Epoch {t+1}\\n-------------------------------\")\n",
    "    train_loop(train_dataloader, model, loss_fn, optimizer)\n",
    "    test_loop(test_dataloader, model, loss_fn)\n",
    "print(\"Done!\")"
   ],
   "id": "b2c4dfbb148cee45",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1\n",
      "-------------------------------\n",
      "loss: 2.295197  [   64/60000]\n",
      "loss: 2.286690  [ 6464/60000]\n",
      "loss: 2.271601  [12864/60000]\n",
      "loss: 2.269881  [19264/60000]\n",
      "loss: 2.243100  [25664/60000]\n",
      "loss: 2.222551  [32064/60000]\n",
      "loss: 2.227910  [38464/60000]\n",
      "loss: 2.196013  [44864/60000]\n",
      "loss: 2.195829  [51264/60000]\n",
      "loss: 2.167131  [57664/60000]\n",
      "Test Error: \n",
      " Accuracy: 46.5%, Avg loss: 2.157610 \n",
      "\n",
      "Epoch 2\n",
      "-------------------------------\n",
      "loss: 2.164239  [   64/60000]\n",
      "loss: 2.159359  [ 6464/60000]\n",
      "loss: 2.101278  [12864/60000]\n",
      "loss: 2.126714  [19264/60000]\n",
      "loss: 2.065790  [25664/60000]\n",
      "loss: 2.011594  [32064/60000]\n",
      "loss: 2.048672  [38464/60000]\n",
      "loss: 1.962424  [44864/60000]\n",
      "loss: 1.969879  [51264/60000]\n",
      "loss: 1.918220  [57664/60000]\n",
      "Test Error: \n",
      " Accuracy: 58.3%, Avg loss: 1.900982 \n",
      "\n",
      "Epoch 3\n",
      "-------------------------------\n",
      "loss: 1.925695  [   64/60000]\n",
      "loss: 1.905023  [ 6464/60000]\n",
      "loss: 1.781793  [12864/60000]\n",
      "loss: 1.841122  [19264/60000]\n",
      "loss: 1.711841  [25664/60000]\n",
      "loss: 1.672258  [32064/60000]\n",
      "loss: 1.706152  [38464/60000]\n",
      "loss: 1.591045  [44864/60000]\n",
      "loss: 1.621228  [51264/60000]\n",
      "loss: 1.537856  [57664/60000]\n",
      "Test Error: \n",
      " Accuracy: 61.1%, Avg loss: 1.535950 \n",
      "\n",
      "Epoch 4\n",
      "-------------------------------\n",
      "loss: 1.594002  [   64/60000]\n",
      "loss: 1.565670  [ 6464/60000]\n",
      "loss: 1.407944  [12864/60000]\n",
      "loss: 1.495741  [19264/60000]\n",
      "loss: 1.355868  [25664/60000]\n",
      "loss: 1.363383  [32064/60000]\n",
      "loss: 1.380506  [38464/60000]\n",
      "loss: 1.291852  [44864/60000]\n",
      "loss: 1.330464  [51264/60000]\n",
      "loss: 1.242680  [57664/60000]\n",
      "Test Error: \n",
      " Accuracy: 63.5%, Avg loss: 1.261330 \n",
      "\n",
      "Epoch 5\n",
      "-------------------------------\n",
      "loss: 1.334339  [   64/60000]\n",
      "loss: 1.320043  [ 6464/60000]\n",
      "loss: 1.150176  [12864/60000]\n",
      "loss: 1.265756  [19264/60000]\n",
      "loss: 1.126287  [25664/60000]\n",
      "loss: 1.161139  [32064/60000]\n",
      "loss: 1.181491  [38464/60000]\n",
      "loss: 1.107484  [44864/60000]\n",
      "loss: 1.150366  [51264/60000]\n",
      "loss: 1.075245  [57664/60000]\n",
      "Test Error: \n",
      " Accuracy: 65.0%, Avg loss: 1.092558 \n",
      "\n",
      "Epoch 6\n",
      "-------------------------------\n",
      "loss: 1.161386  [   64/60000]\n",
      "loss: 1.166636  [ 6464/60000]\n",
      "loss: 0.980756  [12864/60000]\n",
      "loss: 1.125538  [19264/60000]\n",
      "loss: 0.985272  [25664/60000]\n",
      "loss: 1.025389  [32064/60000]\n",
      "loss: 1.061960  [38464/60000]\n",
      "loss: 0.991521  [44864/60000]\n",
      "loss: 1.034498  [51264/60000]\n",
      "loss: 0.973056  [57664/60000]\n",
      "Test Error: \n",
      " Accuracy: 66.1%, Avg loss: 0.984250 \n",
      "\n",
      "Epoch 7\n",
      "-------------------------------\n",
      "loss: 1.041322  [   64/60000]\n",
      "loss: 1.067630  [ 6464/60000]\n",
      "loss: 0.864666  [12864/60000]\n",
      "loss: 1.032918  [19264/60000]\n",
      "loss: 0.895992  [25664/60000]\n",
      "loss: 0.930039  [32064/60000]\n",
      "loss: 0.984452  [38464/60000]\n",
      "loss: 0.916532  [44864/60000]\n",
      "loss: 0.954890  [51264/60000]\n",
      "loss: 0.905240  [57664/60000]\n",
      "Test Error: \n",
      " Accuracy: 67.0%, Avg loss: 0.910472 \n",
      "\n",
      "Epoch 8\n",
      "-------------------------------\n",
      "loss: 0.953052  [   64/60000]\n",
      "loss: 0.998561  [ 6464/60000]\n",
      "loss: 0.781775  [12864/60000]\n",
      "loss: 0.967428  [19264/60000]\n",
      "loss: 0.836386  [25664/60000]\n",
      "loss: 0.860740  [32064/60000]\n",
      "loss: 0.930062  [38464/60000]\n",
      "loss: 0.866929  [44864/60000]\n",
      "loss: 0.897661  [51264/60000]\n",
      "loss: 0.856392  [57664/60000]\n",
      "Test Error: \n",
      " Accuracy: 68.2%, Avg loss: 0.857304 \n",
      "\n",
      "Epoch 9\n",
      "-------------------------------\n",
      "loss: 0.885325  [   64/60000]\n",
      "loss: 0.946455  [ 6464/60000]\n",
      "loss: 0.720075  [12864/60000]\n",
      "loss: 0.918264  [19264/60000]\n",
      "loss: 0.794036  [25664/60000]\n",
      "loss: 0.808838  [32064/60000]\n",
      "loss: 0.888935  [38464/60000]\n",
      "loss: 0.832938  [44864/60000]\n",
      "loss: 0.855130  [51264/60000]\n",
      "loss: 0.818977  [57664/60000]\n",
      "Test Error: \n",
      " Accuracy: 69.3%, Avg loss: 0.817111 \n",
      "\n",
      "Epoch 10\n",
      "-------------------------------\n",
      "loss: 0.831565  [   64/60000]\n",
      "loss: 0.904634  [ 6464/60000]\n",
      "loss: 0.672520  [12864/60000]\n",
      "loss: 0.879969  [19264/60000]\n",
      "loss: 0.761980  [25664/60000]\n",
      "loss: 0.769259  [32064/60000]\n",
      "loss: 0.855768  [38464/60000]\n",
      "loss: 0.808405  [44864/60000]\n",
      "loss: 0.822349  [51264/60000]\n",
      "loss: 0.788789  [57664/60000]\n",
      "Test Error: \n",
      " Accuracy: 70.4%, Avg loss: 0.785278 \n",
      "\n",
      "Done!\n"
     ]
    }
   ],
   "execution_count": 31
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Save and Load the Model",
   "id": "c187f45c2bdce7b2"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T13:15:10.581756Z",
     "start_time": "2024-10-11T13:15:06.217378Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import torch\n",
    "import torchvision.models as models\n",
    "\n",
    "model = models.vgg16(weights='IMAGENET1K_V1')\n",
    "torch.save(model.state_dict(), 'model/model_weights.pth')\n",
    "\n",
    "model = models.vgg16() # we do not specify ``weights``, i.e. create untrained model\n",
    "model.load_state_dict(torch.load('model/model_weights.pth', weights_only=True))\n",
    "model.eval()"
   ],
   "id": "37762b490b09dd7c",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VGG(\n",
       "  (features): Sequential(\n",
       "    (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (1): ReLU(inplace=True)\n",
       "    (2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (3): ReLU(inplace=True)\n",
       "    (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (5): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (6): ReLU(inplace=True)\n",
       "    (7): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (8): ReLU(inplace=True)\n",
       "    (9): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (10): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (11): ReLU(inplace=True)\n",
       "    (12): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (13): ReLU(inplace=True)\n",
       "    (14): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (15): ReLU(inplace=True)\n",
       "    (16): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (17): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (18): ReLU(inplace=True)\n",
       "    (19): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (20): ReLU(inplace=True)\n",
       "    (21): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (22): ReLU(inplace=True)\n",
       "    (23): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (24): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (25): ReLU(inplace=True)\n",
       "    (26): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (27): ReLU(inplace=True)\n",
       "    (28): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (29): ReLU(inplace=True)\n",
       "    (30): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "  )\n",
       "  (avgpool): AdaptiveAvgPool2d(output_size=(7, 7))\n",
       "  (classifier): Sequential(\n",
       "    (0): Linear(in_features=25088, out_features=4096, bias=True)\n",
       "    (1): ReLU(inplace=True)\n",
       "    (2): Dropout(p=0.5, inplace=False)\n",
       "    (3): Linear(in_features=4096, out_features=4096, bias=True)\n",
       "    (4): ReLU(inplace=True)\n",
       "    (5): Dropout(p=0.5, inplace=False)\n",
       "    (6): Linear(in_features=4096, out_features=1000, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 33
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-11T13:16:51.525958Z",
     "start_time": "2024-10-11T13:16:48.907645Z"
    }
   },
   "cell_type": "code",
   "source": [
    "torch.save(model, 'model/model.pth')\n",
    "model = torch.load('model/model.pth', weights_only=False)"
   ],
   "id": "9ad7dfbcb84e2aa3",
   "outputs": [],
   "execution_count": 35
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "478bdbe4d105de5a"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
